[ 484.126858] env[62499]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'linux_bridge' {{(pid=62499) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 484.127206] env[62499]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'noop' {{(pid=62499) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 484.127248] env[62499]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'ovs' {{(pid=62499) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 484.127601] env[62499]: INFO os_vif [-] Loaded VIF plugins: linux_bridge, noop, ovs [ 484.221862] env[62499]: DEBUG oslo_concurrency.processutils [-] Running cmd (subprocess): grep -F node.session.scan /sbin/iscsiadm {{(pid=62499) execute /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/processutils.py:390}} [ 484.232386] env[62499]: DEBUG oslo_concurrency.processutils [-] CMD "grep -F node.session.scan /sbin/iscsiadm" returned: 0 in 0.010s {{(pid=62499) execute /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/processutils.py:428}} [ 484.833614] env[62499]: INFO nova.virt.driver [None req-acb6eed6-5b8b-488f-aa45-0c51e047830a None None] Loading compute driver 'vmwareapi.VMwareVCDriver' [ 484.903785] env[62499]: DEBUG oslo_concurrency.lockutils [-] Acquiring lock "oslo_vmware_api_lock" by "oslo_vmware.api.VMwareAPISession._create_session" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 484.903933] env[62499]: DEBUG oslo_concurrency.lockutils [-] Lock "oslo_vmware_api_lock" acquired by "oslo_vmware.api.VMwareAPISession._create_session" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 484.904055] env[62499]: DEBUG oslo_vmware.service [-] Creating suds client with soap_url='https://vc1.osci.c.eu-de-1.cloud.sap:443/sdk' and wsdl_url='https://vc1.osci.c.eu-de-1.cloud.sap:443/sdk/vimService.wsdl' {{(pid=62499) __init__ /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:242}} [ 488.008889] env[62499]: DEBUG oslo_vmware.service [-] Invoking ServiceInstance.RetrieveServiceContent with opID=oslo.vmware-4e2d697a-fc39-41dd-863b-c01cde80fb96 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 488.024731] env[62499]: DEBUG oslo_vmware.api [-] Logging into host: vc1.osci.c.eu-de-1.cloud.sap. {{(pid=62499) _create_session /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:242}} [ 488.024847] env[62499]: DEBUG oslo_vmware.service [-] Invoking SessionManager.Login with opID=oslo.vmware-868f961a-814f-467b-9779-e7184c80d2b8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 488.049778] env[62499]: INFO oslo_vmware.api [-] Successfully established new session; session ID is fa999. [ 488.049907] env[62499]: DEBUG oslo_concurrency.lockutils [-] Lock "oslo_vmware_api_lock" "released" by "oslo_vmware.api.VMwareAPISession._create_session" :: held 3.146s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 488.050412] env[62499]: INFO nova.virt.vmwareapi.driver [None req-acb6eed6-5b8b-488f-aa45-0c51e047830a None None] VMware vCenter version: 7.0.3 [ 488.053738] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa898936-4b19-4f11-bdb8-ab4a50ea3031 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 488.070669] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0b88959-b31c-4886-8c10-d9d73778070d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 488.076313] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-43f7cd3d-1915-48c4-a138-c933fc50068f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 488.082562] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2e6aebe-462d-41e2-9fc3-832b44e96daf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 488.095121] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2bae36b-d7a0-4078-adcd-ca2134924e23 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 488.100885] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d354461b-dec8-4421-8210-e0f2d3dfc672 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 488.131019] env[62499]: DEBUG oslo_vmware.service [-] Invoking ExtensionManager.FindExtension with opID=oslo.vmware-bf42816a-e9ff-4f3d-86fe-133f54a135d3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 488.136267] env[62499]: DEBUG nova.virt.vmwareapi.driver [None req-acb6eed6-5b8b-488f-aa45-0c51e047830a None None] Extension org.openstack.compute already exists. {{(pid=62499) _register_openstack_extension /opt/stack/nova/nova/virt/vmwareapi/driver.py:226}} [ 488.139016] env[62499]: INFO nova.compute.provider_config [None req-acb6eed6-5b8b-488f-aa45-0c51e047830a None None] No provider configs found in /etc/nova/provider_config/. If files are present, ensure the Nova process has access. [ 488.642271] env[62499]: DEBUG nova.context [None req-acb6eed6-5b8b-488f-aa45-0c51e047830a None None] Found 2 cells: 00000000-0000-0000-0000-000000000000(cell0),8fb5f300-ab58-4d7c-afe8-8d0b4be88dee(cell1) {{(pid=62499) load_cells /opt/stack/nova/nova/context.py:464}} [ 488.644538] env[62499]: DEBUG oslo_concurrency.lockutils [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 488.644805] env[62499]: DEBUG oslo_concurrency.lockutils [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 488.645517] env[62499]: DEBUG oslo_concurrency.lockutils [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 488.645955] env[62499]: DEBUG oslo_concurrency.lockutils [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] Acquiring lock "8fb5f300-ab58-4d7c-afe8-8d0b4be88dee" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 488.646166] env[62499]: DEBUG oslo_concurrency.lockutils [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] Lock "8fb5f300-ab58-4d7c-afe8-8d0b4be88dee" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 488.647260] env[62499]: DEBUG oslo_concurrency.lockutils [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] Lock "8fb5f300-ab58-4d7c-afe8-8d0b4be88dee" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 488.668078] env[62499]: INFO dbcounter [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] Registered counter for database nova_cell0 [ 488.676397] env[62499]: INFO dbcounter [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] Registered counter for database nova_cell1 [ 488.679732] env[62499]: DEBUG oslo_db.sqlalchemy.engines [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=62499) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 488.680367] env[62499]: DEBUG oslo_db.sqlalchemy.engines [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=62499) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 488.684921] env[62499]: ERROR nova.db.main.api [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] No DB access allowed in nova-compute: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 488.684921] env[62499]: result = function(*args, **kwargs) [ 488.684921] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 488.684921] env[62499]: return func(*args, **kwargs) [ 488.684921] env[62499]: File "/opt/stack/nova/nova/context.py", line 422, in gather_result [ 488.684921] env[62499]: result = fn(*args, **kwargs) [ 488.684921] env[62499]: File "/opt/stack/nova/nova/db/main/api.py", line 179, in wrapper [ 488.684921] env[62499]: return f(*args, **kwargs) [ 488.684921] env[62499]: File "/opt/stack/nova/nova/objects/service.py", line 554, in _db_service_get_minimum_version [ 488.684921] env[62499]: return db.service_get_minimum_version(context, binaries) [ 488.684921] env[62499]: File "/opt/stack/nova/nova/db/main/api.py", line 238, in wrapper [ 488.684921] env[62499]: _check_db_access() [ 488.684921] env[62499]: File "/opt/stack/nova/nova/db/main/api.py", line 188, in _check_db_access [ 488.684921] env[62499]: stacktrace = ''.join(traceback.format_stack()) [ 488.684921] env[62499]: [ 488.685732] env[62499]: ERROR nova.db.main.api [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] No DB access allowed in nova-compute: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 488.685732] env[62499]: result = function(*args, **kwargs) [ 488.685732] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 488.685732] env[62499]: return func(*args, **kwargs) [ 488.685732] env[62499]: File "/opt/stack/nova/nova/context.py", line 422, in gather_result [ 488.685732] env[62499]: result = fn(*args, **kwargs) [ 488.685732] env[62499]: File "/opt/stack/nova/nova/db/main/api.py", line 179, in wrapper [ 488.685732] env[62499]: return f(*args, **kwargs) [ 488.685732] env[62499]: File "/opt/stack/nova/nova/objects/service.py", line 554, in _db_service_get_minimum_version [ 488.685732] env[62499]: return db.service_get_minimum_version(context, binaries) [ 488.685732] env[62499]: File "/opt/stack/nova/nova/db/main/api.py", line 238, in wrapper [ 488.685732] env[62499]: _check_db_access() [ 488.685732] env[62499]: File "/opt/stack/nova/nova/db/main/api.py", line 188, in _check_db_access [ 488.685732] env[62499]: stacktrace = ''.join(traceback.format_stack()) [ 488.685732] env[62499]: [ 488.686113] env[62499]: WARNING nova.objects.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] Failed to get minimum service version for cell 00000000-0000-0000-0000-000000000000 [ 488.686327] env[62499]: WARNING nova.objects.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] Failed to get minimum service version for cell 8fb5f300-ab58-4d7c-afe8-8d0b4be88dee [ 488.686695] env[62499]: DEBUG oslo_concurrency.lockutils [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] Acquiring lock "singleton_lock" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 488.686858] env[62499]: DEBUG oslo_concurrency.lockutils [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] Acquired lock "singleton_lock" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 488.687115] env[62499]: DEBUG oslo_concurrency.lockutils [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] Releasing lock "singleton_lock" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 488.687449] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] Full set of CONF: {{(pid=62499) _wait_for_exit_or_signal /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/service.py:363}} [ 488.687592] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ******************************************************************************** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2806}} [ 488.687720] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] Configuration options gathered from: {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2807}} [ 488.687860] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] command line args: ['--config-file', '/etc/nova/nova.conf', '--config-file', '/etc/nova/nova-cpu-common.conf', '--config-file', '/etc/nova/nova-cpu-1.conf'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2808}} [ 488.688077] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] config files: ['/etc/nova/nova.conf', '/etc/nova/nova-cpu-common.conf', '/etc/nova/nova-cpu-1.conf'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2809}} [ 488.688214] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ================================================================================ {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2811}} [ 488.688426] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] allow_resize_to_same_host = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.688611] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] arq_binding_timeout = 300 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.688758] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] backdoor_port = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.688885] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] backdoor_socket = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.689070] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] block_device_allocate_retries = 60 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.689239] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] block_device_allocate_retries_interval = 3 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.689406] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cert = self.pem {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.689573] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] compute_driver = vmwareapi.VMwareVCDriver {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.689738] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] compute_monitors = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.689907] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] config_dir = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.690092] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] config_drive_format = iso9660 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.690237] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] config_file = ['/etc/nova/nova.conf', '/etc/nova/nova-cpu-common.conf', '/etc/nova/nova-cpu-1.conf'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.690406] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] config_source = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.690575] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] console_host = devstack {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.690740] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] control_exchange = nova {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.690897] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cpu_allocation_ratio = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.691070] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] daemon = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.691243] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] debug = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.691403] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] default_access_ip_network_name = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.691568] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] default_availability_zone = nova {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.691725] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] default_ephemeral_format = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.691885] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] default_green_pool_size = 1000 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.692149] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] default_log_levels = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN', 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', 'oslo.messaging=INFO', 'oslo_messaging=INFO', 'iso8601=WARN', 'requests.packages.urllib3.connectionpool=WARN', 'urllib3.connectionpool=WARN', 'websocket=WARN', 'requests.packages.urllib3.util.retry=WARN', 'urllib3.util.retry=WARN', 'keystonemiddleware=WARN', 'routes.middleware=WARN', 'stevedore=WARN', 'taskflow=WARN', 'keystoneauth=WARN', 'oslo.cache=INFO', 'oslo_policy=INFO', 'dogpile.core.dogpile=INFO', 'glanceclient=WARN', 'oslo.privsep.daemon=INFO'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.692323] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] default_schedule_zone = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.692484] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] disk_allocation_ratio = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.692646] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] enable_new_services = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.692824] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] enabled_apis = ['osapi_compute'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.692989] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] enabled_ssl_apis = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.693165] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] flat_injected = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.693324] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] force_config_drive = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.693484] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] force_raw_images = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.693650] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] graceful_shutdown_timeout = 5 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.693809] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] heal_instance_info_cache_interval = 60 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.694033] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] host = cpu-1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.694215] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] initial_cpu_allocation_ratio = 4.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.694381] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] initial_disk_allocation_ratio = 1.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.694546] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] initial_ram_allocation_ratio = 1.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.694758] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] injected_network_template = /opt/stack/nova/nova/virt/interfaces.template {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.694923] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] instance_build_timeout = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.695093] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] instance_delete_interval = 300 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.695262] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] instance_format = [instance: %(uuid)s] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.695429] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] instance_name_template = instance-%08x {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.695585] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] instance_usage_audit = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.695751] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] instance_usage_audit_period = month {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.695914] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] instance_uuid_format = [instance: %(uuid)s] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.696101] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] instances_path = /opt/stack/data/nova/instances {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.696303] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] internal_service_availability_zone = internal {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.696459] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] key = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.696619] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] live_migration_retry_count = 30 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.696786] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] log_color = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.696948] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] log_config_append = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.697127] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] log_date_format = %Y-%m-%d %H:%M:%S {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.697297] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] log_dir = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.697452] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] log_file = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.697579] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] log_options = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.697740] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] log_rotate_interval = 1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.697928] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] log_rotate_interval_type = days {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.698089] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] log_rotation_type = none {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.698228] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] logging_context_format_string = %(color)s%(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(project_name)s %(user_name)s%(color)s] %(instance)s%(color)s%(message)s {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.698356] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] logging_debug_format_suffix = {{(pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d}} {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.698524] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] logging_default_format_string = %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.698710] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] logging_exception_prefix = ERROR %(name)s %(instance)s {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.698842] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.699013] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] long_rpc_timeout = 1800 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.699181] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] max_concurrent_builds = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.699341] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] max_concurrent_live_migrations = 1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.699500] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] max_concurrent_snapshots = 5 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.699658] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] max_local_block_devices = 3 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.699816] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] max_logfile_count = 30 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.699972] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] max_logfile_size_mb = 200 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.700160] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] maximum_instance_delete_attempts = 5 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.700336] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] metadata_listen = 0.0.0.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.700508] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] metadata_listen_port = 8775 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.700676] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] metadata_workers = 2 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.700837] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] migrate_max_retries = -1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.701020] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] mkisofs_cmd = genisoimage {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.701223] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] my_block_storage_ip = 10.180.1.21 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.701429] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] my_ip = 10.180.1.21 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.701524] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] network_allocate_retries = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.701698] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] non_inheritable_image_properties = ['cache_in_nova', 'bittorrent'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.701864] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] osapi_compute_listen = 0.0.0.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.702034] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] osapi_compute_listen_port = 8774 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.702205] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] osapi_compute_unique_server_name_scope = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.702416] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] osapi_compute_workers = 2 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.702529] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] password_length = 12 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.702690] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] periodic_enable = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.702847] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] periodic_fuzzy_delay = 60 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.703014] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] pointer_model = usbtablet {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.703183] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] preallocate_images = none {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.703339] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] publish_errors = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.703469] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] pybasedir = /opt/stack/nova {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.703621] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ram_allocation_ratio = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.703778] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] rate_limit_burst = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.703941] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] rate_limit_except_level = CRITICAL {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.704122] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] rate_limit_interval = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.704287] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] reboot_timeout = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.704444] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] reclaim_instance_interval = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.704596] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] record = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.704759] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] reimage_timeout_per_gb = 60 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.704920] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] report_interval = 120 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.705093] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] rescue_timeout = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.705255] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] reserved_host_cpus = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.705414] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] reserved_host_disk_mb = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.705570] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] reserved_host_memory_mb = 512 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.705723] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] reserved_huge_pages = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.705880] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] resize_confirm_window = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.706046] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] resize_fs_using_block_device = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.706205] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] resume_guests_state_on_host_boot = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.706396] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] rootwrap_config = /etc/nova/rootwrap.conf {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.706567] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] rpc_response_timeout = 60 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.706727] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] run_external_periodic_tasks = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.706891] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] running_deleted_instance_action = reap {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.707060] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] running_deleted_instance_poll_interval = 1800 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.707224] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] running_deleted_instance_timeout = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.707401] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] scheduler_instance_sync_interval = 120 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.707583] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] service_down_time = 720 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.707751] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] servicegroup_driver = db {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.707904] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] shell_completion = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.708094] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] shelved_offload_time = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.708262] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] shelved_poll_interval = 3600 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.708432] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] shutdown_timeout = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.708599] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] source_is_ipv6 = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.708761] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ssl_only = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.709011] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] state_path = /opt/stack/data/n-cpu-1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.709187] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] sync_power_state_interval = 600 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.709348] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] sync_power_state_pool_size = 1000 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.709515] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] syslog_log_facility = LOG_USER {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.709671] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] tempdir = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.709827] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] timeout_nbd = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.709990] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] transport_url = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.710160] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] update_resources_interval = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.710318] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] use_cow_images = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.710475] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] use_eventlog = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.710625] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] use_journal = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.710780] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] use_json = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.710932] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] use_rootwrap_daemon = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.711101] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] use_stderr = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.711263] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] use_syslog = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.711418] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vcpu_pin_set = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.711583] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vif_plugging_is_fatal = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.711745] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vif_plugging_timeout = 300 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.711908] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] virt_mkfs = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.712093] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] volume_usage_poll_interval = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.712260] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] watch_log_file = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.712432] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] web = /usr/share/spice-html5 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 488.712615] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_brick.lock_path = /opt/stack/data/n-cpu-1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.712778] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_brick.wait_mpath_device_attempts = 4 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.712941] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_brick.wait_mpath_device_interval = 1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.713124] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_concurrency.disable_process_locking = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.713646] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_concurrency.lock_path = /opt/stack/data/n-cpu-1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.713842] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_metrics.metrics_buffer_size = 1000 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.714030] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_metrics.metrics_enabled = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.714214] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_metrics.metrics_process_name = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.714389] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_metrics.metrics_socket_file = /var/tmp/metrics_collector.sock {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.714559] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_metrics.metrics_thread_stop_timeout = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.714746] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.auth_strategy = keystone {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.714913] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.compute_link_prefix = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.715101] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.config_drive_skip_versions = 1.0 2007-01-19 2007-03-01 2007-08-29 2007-10-10 2007-12-15 2008-02-01 2008-09-01 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.715283] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.dhcp_domain = novalocal {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.715458] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.enable_instance_password = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.715631] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.glance_link_prefix = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.715801] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.instance_list_cells_batch_fixed_size = 100 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.715977] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.instance_list_cells_batch_strategy = distributed {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.716158] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.instance_list_per_project_cells = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.716350] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.list_records_by_skipping_down_cells = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.716533] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.local_metadata_per_cell = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.716710] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.max_limit = 1000 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.716882] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.metadata_cache_expiration = 15 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.717081] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.neutron_default_tenant_id = default {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.717279] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.response_validation = warn {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.717487] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.use_neutron_default_nets = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.717670] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.vendordata_dynamic_connect_timeout = 5 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.717839] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.vendordata_dynamic_failure_fatal = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.718025] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.vendordata_dynamic_read_timeout = 5 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.718202] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.vendordata_dynamic_ssl_certfile = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.718376] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.vendordata_dynamic_targets = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.718544] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.vendordata_jsonfile_path = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.718727] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api.vendordata_providers = ['StaticJSON'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.718917] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.backend = dogpile.cache.memcached {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.719096] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.backend_argument = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.719273] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.config_prefix = cache.oslo {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.719447] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.dead_timeout = 60.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.719612] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.debug_cache_backend = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.719775] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.enable_retry_client = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.719938] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.enable_socket_keepalive = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.720121] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.enabled = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.720289] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.enforce_fips_mode = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.720455] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.expiration_time = 600 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.720621] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.hashclient_retry_attempts = 2 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.720785] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.hashclient_retry_delay = 1.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.720951] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.memcache_dead_retry = 300 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.721123] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.memcache_password = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.721290] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.memcache_pool_connection_get_timeout = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.721456] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.memcache_pool_flush_on_reconnect = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.721618] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.memcache_pool_maxsize = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.721780] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.memcache_pool_unused_timeout = 60 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.721941] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.memcache_sasl_enabled = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.722166] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.memcache_servers = ['localhost:11211'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.722323] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.memcache_socket_timeout = 1.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.722487] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.memcache_username = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.722648] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.proxies = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.722812] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.redis_db = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.722969] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.redis_password = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.723153] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.redis_sentinel_service_name = mymaster {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.723332] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.redis_sentinels = ['localhost:26379'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.723502] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.redis_server = localhost:6379 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.723667] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.redis_socket_timeout = 1.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.723827] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.redis_username = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.723990] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.retry_attempts = 2 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.724181] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.retry_delay = 0.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.724359] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.socket_keepalive_count = 1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.724529] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.socket_keepalive_idle = 1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.724690] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.socket_keepalive_interval = 1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.724847] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.tls_allowed_ciphers = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.725010] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.tls_cafile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.725177] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.tls_certfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.725338] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.tls_enabled = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.725497] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cache.tls_keyfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.725667] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cinder.auth_section = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.725840] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cinder.auth_type = password {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.726006] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cinder.cafile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.726205] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cinder.catalog_info = volumev3::publicURL {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.726412] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cinder.certfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.726596] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cinder.collect_timing = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.726762] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cinder.cross_az_attach = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.726926] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cinder.debug = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.727101] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cinder.endpoint_template = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.727294] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cinder.http_retries = 3 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.727504] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cinder.insecure = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.727674] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cinder.keyfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.727846] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cinder.os_region_name = RegionOne {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.728019] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cinder.split_loggers = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.728227] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cinder.timeout = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.728355] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] compute.consecutive_build_service_disable_threshold = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.728517] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] compute.cpu_dedicated_set = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.728700] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] compute.cpu_shared_set = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.728877] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] compute.image_type_exclude_list = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.729051] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] compute.live_migration_wait_for_vif_plug = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.729220] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] compute.max_concurrent_disk_ops = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.729388] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] compute.max_disk_devices_to_attach = -1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.729552] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] compute.packing_host_numa_cells_allocation_strategy = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.729722] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] compute.provider_config_location = /etc/nova/provider_config/ {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.729884] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] compute.resource_provider_association_refresh = 300 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.730056] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] compute.sharing_providers_max_uuids_per_request = 200 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.730223] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] compute.shutdown_retry_interval = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.730405] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] compute.vmdk_allowed_types = ['streamOptimized', 'monolithicSparse'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.730593] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] conductor.workers = 2 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.730768] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] console.allowed_origins = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.730929] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] console.ssl_ciphers = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.731127] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] console.ssl_minimum_version = default {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.731305] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] consoleauth.enforce_session_timeout = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.731477] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] consoleauth.token_ttl = 600 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.731646] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.cafile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.731805] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.certfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.731968] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.collect_timing = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.732144] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.connect_retries = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.732305] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.connect_retry_delay = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.732464] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.endpoint_override = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.732626] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.insecure = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.732784] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.keyfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.732945] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.max_version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.733114] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.min_version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.733276] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.region_name = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.733434] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.retriable_status_codes = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.733640] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.service_name = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.733823] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.service_type = accelerator {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.733986] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.split_loggers = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.734163] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.status_code_retries = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.734324] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.status_code_retry_delay = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.734484] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.timeout = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.734664] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.valid_interfaces = ['internal', 'public'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.734825] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] cyborg.version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.735009] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.backend = sqlalchemy {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.735188] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.connection = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.735356] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.connection_debug = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.735525] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.connection_parameters = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.735749] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.connection_recycle_time = 3600 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.735995] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.connection_trace = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.736217] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.db_inc_retry_interval = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.736419] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.db_max_retries = 20 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.736597] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.db_max_retry_interval = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.736765] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.db_retry_interval = 1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.736930] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.max_overflow = 50 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.737107] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.max_pool_size = 5 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.737276] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.max_retries = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.737448] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.mysql_sql_mode = TRADITIONAL {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.737610] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.mysql_wsrep_sync_wait = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.737769] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.pool_timeout = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.737930] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.retry_interval = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.738100] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.slave_connection = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.738266] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.sqlite_synchronous = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.738494] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] database.use_db_reconnect = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.738734] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.backend = sqlalchemy {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.738943] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.connection = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.739187] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.connection_debug = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.739393] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.connection_parameters = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.739569] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.connection_recycle_time = 3600 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.739736] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.connection_trace = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.739899] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.db_inc_retry_interval = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.740087] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.db_max_retries = 20 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.740263] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.db_max_retry_interval = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.740430] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.db_retry_interval = 1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.740594] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.max_overflow = 50 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.740792] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.max_pool_size = 5 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.741017] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.max_retries = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.741249] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.mysql_sql_mode = TRADITIONAL {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.741460] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.mysql_wsrep_sync_wait = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.741699] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.pool_timeout = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.741895] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.retry_interval = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.742076] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.slave_connection = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.742315] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] api_database.sqlite_synchronous = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.742550] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] devices.enabled_mdev_types = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.742795] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ephemeral_storage_encryption.cipher = aes-xts-plain64 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.743034] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ephemeral_storage_encryption.default_format = luks {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.743229] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ephemeral_storage_encryption.enabled = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.743403] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ephemeral_storage_encryption.key_size = 512 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.743580] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.api_servers = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.743746] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.cafile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.743908] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.certfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.744102] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.collect_timing = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.744274] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.connect_retries = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.744437] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.connect_retry_delay = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.744600] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.debug = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.744768] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.default_trusted_certificate_ids = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.744932] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.enable_certificate_validation = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.745108] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.enable_rbd_download = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.745271] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.endpoint_override = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.745440] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.insecure = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.745604] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.keyfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.745764] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.max_version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.745989] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.min_version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.746192] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.num_retries = 3 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.746392] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.rbd_ceph_conf = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.746567] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.rbd_connect_timeout = 5 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.746743] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.rbd_pool = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.746914] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.rbd_user = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.747089] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.region_name = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.747266] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.retriable_status_codes = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.747498] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.service_name = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.747723] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.service_type = image {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.747922] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.split_loggers = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.748136] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.status_code_retries = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.748322] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.status_code_retry_delay = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.748499] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.timeout = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.748693] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.valid_interfaces = ['internal', 'public'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.748861] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.verify_glance_signatures = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.749042] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] glance.version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.749227] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] guestfs.debug = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.749414] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] mks.enabled = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.749777] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] mks.mksproxy_base_url = http://127.0.0.1:6090/ {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.749970] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] image_cache.manager_interval = 2400 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.750158] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] image_cache.precache_concurrency = 1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.750334] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] image_cache.remove_unused_base_images = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.750511] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] image_cache.remove_unused_original_minimum_age_seconds = 86400 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.750683] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] image_cache.remove_unused_resized_minimum_age_seconds = 3600 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.750894] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] image_cache.subdirectory_name = _base {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.751048] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.api_max_retries = 60 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.751218] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.api_retry_interval = 2 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.751379] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.auth_section = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.751543] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.auth_type = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.751704] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.cafile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.751862] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.certfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.752033] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.collect_timing = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.752202] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.conductor_group = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.752361] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.connect_retries = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.752523] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.connect_retry_delay = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.752701] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.endpoint_override = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.752839] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.insecure = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.753010] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.keyfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.753183] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.max_version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.753342] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.min_version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.753509] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.peer_list = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.753665] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.region_name = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.753822] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.retriable_status_codes = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.753983] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.serial_console_state_timeout = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.754157] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.service_name = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.754387] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.service_type = baremetal {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.754574] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.shard = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.754741] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.split_loggers = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.754903] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.status_code_retries = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.755126] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.status_code_retry_delay = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.755256] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.timeout = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.755441] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.valid_interfaces = ['internal', 'public'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.755609] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ironic.version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.755794] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] key_manager.backend = nova.keymgr.conf_key_mgr.ConfKeyManager {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.755968] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] key_manager.fixed_key = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.756167] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.auth_endpoint = http://localhost/identity/v3 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.756357] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.barbican_api_version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.756532] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.barbican_endpoint = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.756707] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.barbican_endpoint_type = public {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.756866] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.barbican_region_name = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.757047] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.cafile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.757218] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.certfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.757417] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.collect_timing = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.757587] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.insecure = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.757749] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.keyfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.757912] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.number_of_retries = 60 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.758088] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.retry_delay = 1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.758259] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.send_service_user_token = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.758425] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.split_loggers = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.758584] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.timeout = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.758745] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.verify_ssl = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.758901] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican.verify_ssl_path = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.759081] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican_service_user.auth_section = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.759248] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican_service_user.auth_type = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.759405] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican_service_user.cafile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.759566] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican_service_user.certfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.759727] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican_service_user.collect_timing = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.759889] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican_service_user.insecure = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.760055] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican_service_user.keyfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.760221] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican_service_user.split_loggers = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.760381] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] barbican_service_user.timeout = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.760548] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vault.approle_role_id = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.760706] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vault.approle_secret_id = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.760876] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vault.kv_mountpoint = secret {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.761055] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vault.kv_path = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.761227] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vault.kv_version = 2 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.761389] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vault.namespace = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.761548] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vault.root_token_id = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.761705] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vault.ssl_ca_crt_file = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.761873] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vault.timeout = 60.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.762057] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vault.use_ssl = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.762237] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vault.vault_url = http://127.0.0.1:8200 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.762409] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.auth_section = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.762574] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.auth_type = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.762733] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.cafile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.762893] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.certfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.763066] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.collect_timing = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.763226] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.connect_retries = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.763384] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.connect_retry_delay = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.763543] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.endpoint_override = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.763703] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.insecure = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.763858] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.keyfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.764025] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.max_version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.764178] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.min_version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.764334] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.region_name = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.764493] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.retriable_status_codes = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.764648] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.service_name = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.764815] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.service_type = identity {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.764975] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.split_loggers = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.765146] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.status_code_retries = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.765310] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.status_code_retry_delay = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.765471] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.timeout = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.765651] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.valid_interfaces = ['internal', 'public'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.765810] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] keystone.version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.766014] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.connection_uri = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.766204] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.cpu_mode = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.766403] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.cpu_model_extra_flags = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.766585] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.cpu_models = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.766758] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.cpu_power_governor_high = performance {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.766927] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.cpu_power_governor_low = powersave {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.767113] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.cpu_power_management = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.767315] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.cpu_power_management_strategy = cpu_state {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.767521] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.device_detach_attempts = 8 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.767692] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.device_detach_timeout = 20 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.767861] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.disk_cachemodes = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.768032] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.disk_prefix = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.768203] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.enabled_perf_events = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.768368] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.file_backed_memory = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.768535] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.gid_maps = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.768693] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.hw_disk_discard = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.768850] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.hw_machine_type = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.769031] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.images_rbd_ceph_conf = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.769209] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.images_rbd_glance_copy_poll_interval = 15 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.769386] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.images_rbd_glance_copy_timeout = 600 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.769560] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.images_rbd_glance_store_name = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.769730] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.images_rbd_pool = rbd {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.769901] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.images_type = default {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.770085] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.images_volume_group = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.770259] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.inject_key = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.770427] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.inject_partition = -2 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.770586] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.inject_password = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.770747] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.iscsi_iface = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.770907] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.iser_use_multipath = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.771083] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.live_migration_bandwidth = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.771250] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.live_migration_completion_timeout = 800 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.771414] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.live_migration_downtime = 500 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.771577] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.live_migration_downtime_delay = 75 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.771736] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.live_migration_downtime_steps = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.771896] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.live_migration_inbound_addr = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.772067] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.live_migration_permit_auto_converge = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.772244] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.live_migration_permit_post_copy = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.772409] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.live_migration_scheme = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.772583] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.live_migration_timeout_action = abort {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.772746] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.live_migration_tunnelled = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.772945] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.live_migration_uri = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.773077] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.live_migration_with_native_tls = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.773240] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.max_queues = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.773402] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.mem_stats_period_seconds = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.773639] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.migration_inbound_addr = 10.180.1.21 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.773801] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.nfs_mount_options = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.774133] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.nfs_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.774320] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.num_aoe_discover_tries = 3 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.774491] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.num_iser_scan_tries = 5 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.774655] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.num_memory_encrypted_guests = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.774820] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.num_nvme_discover_tries = 5 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.774985] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.num_pcie_ports = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.775173] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.num_volume_scan_tries = 5 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.775343] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.pmem_namespaces = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.775508] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.quobyte_client_cfg = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.775795] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.quobyte_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.775970] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.rbd_connect_timeout = 5 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.776157] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.rbd_destroy_volume_retries = 12 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.776351] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.rbd_destroy_volume_retry_interval = 5 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.776530] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.rbd_secret_uuid = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.776693] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.rbd_user = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.776878] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.realtime_scheduler_priority = 1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.777073] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.remote_filesystem_transport = ssh {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.777246] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.rescue_image_id = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.777453] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.rescue_kernel_id = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.777677] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.rescue_ramdisk_id = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.777855] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.rng_dev_path = /dev/urandom {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.778039] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.rx_queue_size = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.778223] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.smbfs_mount_options = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.778502] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.smbfs_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.778679] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.snapshot_compression = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.778844] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.snapshot_image_format = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.779075] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.snapshots_directory = /opt/stack/data/nova/instances/snapshots {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.779248] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.sparse_logical_volumes = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.779413] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.swtpm_enabled = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.779583] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.swtpm_group = tss {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.779748] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.swtpm_user = tss {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.779915] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.sysinfo_serial = unique {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.780088] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.tb_cache_size = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.780253] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.tx_queue_size = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.780421] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.uid_maps = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.780584] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.use_virtio_for_bridges = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.780755] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.virt_type = kvm {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.780924] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.volume_clear = zero {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.781106] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.volume_clear_size = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.781275] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.volume_use_multipath = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.781435] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.vzstorage_cache_path = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.781605] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.vzstorage_log_path = /var/log/vstorage/%(cluster_name)s/nova.log.gz {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.781773] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.vzstorage_mount_group = qemu {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.781938] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.vzstorage_mount_opts = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.782133] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.vzstorage_mount_perms = 0770 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.782416] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.vzstorage_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.782599] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.vzstorage_mount_user = stack {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.782769] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] libvirt.wait_soft_reboot_seconds = 120 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.782946] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.auth_section = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.783138] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.auth_type = password {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.783303] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.cafile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.783466] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.certfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.783631] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.collect_timing = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.783826] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.connect_retries = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.784045] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.connect_retry_delay = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.784286] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.default_floating_pool = public {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.784465] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.endpoint_override = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.784638] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.extension_sync_interval = 600 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.784805] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.http_retries = 3 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.784971] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.insecure = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.785147] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.keyfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.785311] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.max_version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.785481] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.metadata_proxy_shared_secret = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.785641] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.min_version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.785809] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.ovs_bridge = br-int {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.785976] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.physnets = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.786176] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.region_name = RegionOne {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.786371] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.retriable_status_codes = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.786558] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.service_metadata_proxy = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.786723] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.service_name = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.786895] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.service_type = network {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.787072] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.split_loggers = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.787240] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.status_code_retries = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.787456] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.status_code_retry_delay = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.787636] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.timeout = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.787825] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.valid_interfaces = ['internal', 'public'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.787991] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] neutron.version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.788184] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] notifications.bdms_in_notifications = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.788364] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] notifications.default_level = INFO {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.788540] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] notifications.notification_format = unversioned {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.788735] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] notifications.notify_on_state_change = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.788919] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] notifications.versioned_notifications_topics = ['versioned_notifications'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.789108] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] pci.alias = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.789284] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] pci.device_spec = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.789453] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] pci.report_in_placement = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.789625] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.auth_section = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.789799] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.auth_type = password {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.789966] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.auth_url = http://10.180.1.21/identity {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.790155] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.cafile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.790319] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.certfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.790484] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.collect_timing = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.790643] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.connect_retries = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.790802] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.connect_retry_delay = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.790959] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.default_domain_id = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.791132] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.default_domain_name = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.791293] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.domain_id = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.791451] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.domain_name = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.791606] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.endpoint_override = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.791767] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.insecure = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.791923] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.keyfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.792089] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.max_version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.792251] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.min_version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.792420] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.password = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.792579] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.project_domain_id = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.792744] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.project_domain_name = Default {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.792913] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.project_id = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.793097] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.project_name = service {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.793272] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.region_name = RegionOne {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.793434] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.retriable_status_codes = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.793593] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.service_name = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.793823] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.service_type = placement {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.794036] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.split_loggers = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.794218] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.status_code_retries = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.794382] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.status_code_retry_delay = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.794545] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.system_scope = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.794702] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.timeout = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.794860] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.trust_id = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.795028] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.user_domain_id = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.795205] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.user_domain_name = Default {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.795367] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.user_id = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.795541] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.username = nova {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.795723] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.valid_interfaces = ['internal', 'public'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.795885] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] placement.version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.796076] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] quota.cores = 20 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.796248] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] quota.count_usage_from_placement = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.796447] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] quota.driver = nova.quota.DbQuotaDriver {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.796633] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] quota.injected_file_content_bytes = 10240 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.796800] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] quota.injected_file_path_length = 255 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.796966] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] quota.injected_files = 5 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.797149] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] quota.instances = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.797342] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] quota.key_pairs = 100 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.797545] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] quota.metadata_items = 128 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.797718] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] quota.ram = 51200 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.797885] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] quota.recheck_quota = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.798078] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] quota.server_group_members = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.798258] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] quota.server_groups = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.798443] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] scheduler.discover_hosts_in_cells_interval = -1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.798610] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] scheduler.enable_isolated_aggregate_filtering = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.798772] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] scheduler.image_metadata_prefilter = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.798969] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] scheduler.limit_tenants_to_placement_aggregate = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.799111] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] scheduler.max_attempts = 3 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.799279] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] scheduler.max_placement_results = 1000 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.799444] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] scheduler.placement_aggregate_required_for_tenants = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.799607] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] scheduler.query_placement_for_image_type_support = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.799767] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] scheduler.query_placement_for_routed_network_aggregates = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.799940] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] scheduler.workers = 2 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.800125] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.aggregate_image_properties_isolation_namespace = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.800300] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.aggregate_image_properties_isolation_separator = . {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.800481] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.available_filters = ['nova.scheduler.filters.all_filters'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.800649] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.build_failure_weight_multiplier = 1000000.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.800815] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.cpu_weight_multiplier = 1.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.800977] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.cross_cell_move_weight_multiplier = 1000000.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.801157] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.disk_weight_multiplier = 1.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.801378] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.enabled_filters = ['ComputeFilter', 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter', 'ServerGroupAntiAffinityFilter', 'ServerGroupAffinityFilter', 'SameHostFilter', 'DifferentHostFilter'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.801516] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.host_subset_size = 1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.801685] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.hypervisor_version_weight_multiplier = 1.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.801846] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.image_properties_default_architecture = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.802015] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.io_ops_weight_multiplier = -1.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.802208] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.isolated_hosts = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.802376] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.isolated_images = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.802543] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.max_instances_per_host = 50 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.802705] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.max_io_ops_per_host = 8 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.802870] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.num_instances_weight_multiplier = 0.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.803042] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.pci_in_placement = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.803213] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.pci_weight_multiplier = 1.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.803380] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.ram_weight_multiplier = 1.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.803544] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.restrict_isolated_hosts_to_isolated_images = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.803704] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.shuffle_best_same_weighed_hosts = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.803867] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.soft_affinity_weight_multiplier = 1.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.804038] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.soft_anti_affinity_weight_multiplier = 1.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.804204] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.track_instance_changes = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.804380] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] filter_scheduler.weight_classes = ['nova.scheduler.weights.all_weighers'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.804555] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] metrics.required = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.804721] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] metrics.weight_multiplier = 1.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.804885] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] metrics.weight_of_unavailable = -10000.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.805059] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] metrics.weight_setting = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.805377] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] serial_console.base_url = ws://127.0.0.1:6083/ {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.805556] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] serial_console.enabled = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.805736] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] serial_console.port_range = 10000:20000 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.805908] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] serial_console.proxyclient_address = 127.0.0.1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.806102] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] serial_console.serialproxy_host = 0.0.0.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.806302] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] serial_console.serialproxy_port = 6083 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.806483] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] service_user.auth_section = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.806660] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] service_user.auth_type = password {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.806823] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] service_user.cafile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.806983] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] service_user.certfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.807164] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] service_user.collect_timing = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.807353] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] service_user.insecure = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.807582] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] service_user.keyfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.807770] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] service_user.send_service_user_token = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.807939] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] service_user.split_loggers = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.808132] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] service_user.timeout = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.808309] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] spice.agent_enabled = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.808476] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] spice.enabled = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.808789] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] spice.html5proxy_base_url = http://127.0.0.1:6082/spice_auto.html {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.808982] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] spice.html5proxy_host = 0.0.0.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.809172] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] spice.html5proxy_port = 6082 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.809337] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] spice.image_compression = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.809495] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] spice.jpeg_compression = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.809654] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] spice.playback_compression = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.809816] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] spice.require_secure = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.809985] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] spice.server_listen = 127.0.0.1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.810168] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] spice.server_proxyclient_address = 127.0.0.1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.810329] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] spice.streaming_mode = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.810489] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] spice.zlib_compression = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.810652] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] upgrade_levels.baseapi = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.810821] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] upgrade_levels.compute = auto {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.811008] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] upgrade_levels.conductor = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.811222] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] upgrade_levels.scheduler = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.811398] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vendordata_dynamic_auth.auth_section = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.811566] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vendordata_dynamic_auth.auth_type = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.811725] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vendordata_dynamic_auth.cafile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.811884] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vendordata_dynamic_auth.certfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.812060] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vendordata_dynamic_auth.collect_timing = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.812228] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vendordata_dynamic_auth.insecure = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.812390] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vendordata_dynamic_auth.keyfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.812554] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vendordata_dynamic_auth.split_loggers = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.812715] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vendordata_dynamic_auth.timeout = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.812892] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.api_retry_count = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.813064] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.ca_file = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.813239] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.cache_prefix = devstack-image-cache {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.813423] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.cluster_name = testcl1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.813568] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.connection_pool_size = 10 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.813724] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.console_delay_seconds = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.813893] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.datastore_regex = ^datastore.* {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.814112] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.host_ip = vc1.osci.c.eu-de-1.cloud.sap {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.814288] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.host_password = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.814457] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.host_port = 443 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.814624] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.host_username = administrator@vsphere.local {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.814792] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.insecure = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.814953] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.integration_bridge = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.815131] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.maximum_objects = 100 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.815295] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.pbm_default_policy = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.815459] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.pbm_enabled = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.815617] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.pbm_wsdl_location = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.815786] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.serial_log_dir = /opt/vmware/vspc {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.815944] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.serial_port_proxy_uri = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.816128] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.serial_port_service_uri = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.816327] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.task_poll_interval = 0.5 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.816520] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.use_linked_clone = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.816965] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.vnc_keymap = en-us {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.816965] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.vnc_port = 5900 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.817044] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vmware.vnc_port_total = 10000 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.817201] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vnc.auth_schemes = ['none'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.817425] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vnc.enabled = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.817735] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vnc.novncproxy_base_url = http://127.0.0.1:6080/vnc_auto.html {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.817925] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vnc.novncproxy_host = 0.0.0.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.818116] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vnc.novncproxy_port = 6080 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.818300] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vnc.server_listen = 127.0.0.1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.818478] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vnc.server_proxyclient_address = 127.0.0.1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.818644] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vnc.vencrypt_ca_certs = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.818802] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vnc.vencrypt_client_cert = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.818960] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vnc.vencrypt_client_key = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.819196] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.disable_compute_service_check_for_ffu = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.819325] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.disable_deep_image_inspection = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.819491] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.disable_fallback_pcpu_query = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.819654] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.disable_group_policy_check_upcall = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.819816] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.disable_libvirt_livesnapshot = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.819977] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.disable_rootwrap = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.820171] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.enable_numa_live_migration = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.820339] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.enable_qemu_monitor_announce_self = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.820502] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.ensure_libvirt_rbd_instance_dir_cleanup = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.820664] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.handle_virt_lifecycle_events = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.820823] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.libvirt_disable_apic = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.820983] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.never_download_image_if_on_rbd = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.821162] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.qemu_monitor_announce_self_count = 3 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.821324] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.qemu_monitor_announce_self_interval = 1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.821497] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.reserve_disk_resource_for_image_cache = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.821661] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.skip_cpu_compare_at_startup = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.821821] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.skip_cpu_compare_on_dest = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.821980] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.skip_hypervisor_version_check_on_lm = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.822153] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.skip_reserve_in_use_ironic_nodes = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.822314] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.unified_limits_count_pcpu_as_vcpu = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.822482] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] workarounds.wait_for_vif_plugged_event_during_hard_reboot = [] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.822665] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] wsgi.api_paste_config = /etc/nova/api-paste.ini {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.822835] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] wsgi.client_socket_timeout = 900 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.823007] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] wsgi.default_pool_size = 1000 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.823182] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] wsgi.keep_alive = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.823350] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] wsgi.max_header_line = 16384 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.823533] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] wsgi.secure_proxy_ssl_header = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.823669] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] wsgi.ssl_ca_file = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.823828] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] wsgi.ssl_cert_file = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.823985] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] wsgi.ssl_key_file = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.824163] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] wsgi.tcp_keepidle = 600 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.824351] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] wsgi.wsgi_log_format = %(client_ip)s "%(request_line)s" status: %(status_code)s len: %(body_length)s time: %(wall_seconds).7f {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.824531] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] zvm.ca_file = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.824695] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] zvm.cloud_connector_url = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.824978] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] zvm.image_tmp_path = /opt/stack/data/n-cpu-1/images {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.825183] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] zvm.reachable_timeout = 300 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.825369] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_policy.enforce_new_defaults = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.825757] env[62499]: WARNING oslo_config.cfg [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] Deprecated: Option "enforce_scope" from group "oslo_policy" is deprecated for removal (This configuration was added temporarily to facilitate a smooth transition to the new RBAC. OpenStack will always enforce scope checks. This configuration option is deprecated and will be removed in the 2025.2 cycle.). Its value may be silently ignored in the future. [ 488.825940] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_policy.enforce_scope = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.826134] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_policy.policy_default_rule = default {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.826348] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_policy.policy_dirs = ['policy.d'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.826543] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_policy.policy_file = policy.yaml {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.826724] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_policy.remote_content_type = application/x-www-form-urlencoded {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.826887] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_policy.remote_ssl_ca_crt_file = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.827061] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_policy.remote_ssl_client_crt_file = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.827225] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_policy.remote_ssl_client_key_file = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.827430] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_policy.remote_ssl_verify_server_crt = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.827617] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_versionedobjects.fatal_exception_format_errors = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.827797] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_middleware.http_basic_auth_user_file = /etc/htpasswd {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.827973] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] profiler.connection_string = messaging:// {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.828161] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] profiler.enabled = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.828333] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] profiler.es_doc_type = notification {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.828502] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] profiler.es_scroll_size = 10000 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.828672] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] profiler.es_scroll_time = 2m {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.828835] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] profiler.filter_error_trace = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.829011] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] profiler.hmac_keys = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.829203] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] profiler.sentinel_service_name = mymaster {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.829377] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] profiler.socket_timeout = 0.1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.829543] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] profiler.trace_requests = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.829703] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] profiler.trace_sqlalchemy = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.829884] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] profiler_jaeger.process_tags = {} {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.830057] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] profiler_jaeger.service_name_prefix = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.830224] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] profiler_otlp.service_name_prefix = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.830392] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] remote_debug.host = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.830556] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] remote_debug.port = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.830736] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.amqp_auto_delete = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.830933] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.amqp_durable_queues = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.831113] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.conn_pool_min_size = 2 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.831282] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.conn_pool_ttl = 1200 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.831448] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.direct_mandatory_flag = True {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.831611] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.enable_cancel_on_failover = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.831773] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.heartbeat_in_pthread = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.831934] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.heartbeat_rate = 3 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.832110] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.heartbeat_timeout_threshold = 60 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.832286] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.hostname = devstack {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.832448] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.kombu_compression = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.832621] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.kombu_failover_strategy = round-robin {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.832789] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.kombu_missing_consumer_retry_timeout = 60 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.832960] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.kombu_reconnect_delay = 1.0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.833162] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.processname = nova-compute {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.833336] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.rabbit_ha_queues = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.833503] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.rabbit_interval_max = 30 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.833677] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.rabbit_login_method = AMQPLAIN {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.833842] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.rabbit_qos_prefetch_count = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.834022] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.rabbit_quorum_delivery_limit = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.834188] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_bytes = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.834355] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_length = 0 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.834521] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.rabbit_quorum_queue = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.834687] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.rabbit_retry_backoff = 2 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.834848] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.rabbit_retry_interval = 1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.835016] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.rabbit_stream_fanout = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.835188] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.rabbit_transient_queues_ttl = 1800 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.835351] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.rabbit_transient_quorum_queue = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.835520] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.rpc_conn_pool_size = 30 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.835686] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.ssl = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.835858] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.ssl_ca_file = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.836036] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.ssl_cert_file = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.836203] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.ssl_enforce_fips_mode = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.836408] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.ssl_key_file = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.836592] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.ssl_version = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.836758] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_rabbit.use_queue_manager = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.836948] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_notifications.driver = ['messagingv2'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.837147] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_notifications.retry = -1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.837341] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_notifications.topics = ['notifications'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.837548] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_messaging_notifications.transport_url = **** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.837728] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.auth_section = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.837893] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.auth_type = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.838066] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.cafile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.838230] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.certfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.838394] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.collect_timing = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.838557] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.connect_retries = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.838748] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.connect_retry_delay = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.838915] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.endpoint_id = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.839084] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.endpoint_override = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.839248] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.insecure = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.839406] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.keyfile = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.839560] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.max_version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.839719] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.min_version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.839874] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.region_name = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.840043] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.retriable_status_codes = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.840206] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.service_name = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.840363] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.service_type = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.840528] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.split_loggers = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.840687] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.status_code_retries = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.840843] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.status_code_retry_delay = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.841006] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.timeout = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.841173] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.valid_interfaces = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.841332] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_limit.version = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.841500] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_reports.file_event_handler = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.841665] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_reports.file_event_handler_interval = 1 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.841826] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] oslo_reports.log_dir = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.841996] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vif_plug_linux_bridge_privileged.capabilities = [12] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.842188] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vif_plug_linux_bridge_privileged.group = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.842350] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vif_plug_linux_bridge_privileged.helper_command = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.842516] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vif_plug_linux_bridge_privileged.logger_name = oslo_privsep.daemon {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.842679] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vif_plug_linux_bridge_privileged.thread_pool_size = 8 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.842835] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vif_plug_linux_bridge_privileged.user = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.843010] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vif_plug_ovs_privileged.capabilities = [12, 1] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.843176] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vif_plug_ovs_privileged.group = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.843343] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vif_plug_ovs_privileged.helper_command = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.843512] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vif_plug_ovs_privileged.logger_name = oslo_privsep.daemon {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.843673] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vif_plug_ovs_privileged.thread_pool_size = 8 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.843831] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] vif_plug_ovs_privileged.user = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.844056] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_vif_linux_bridge.flat_interface = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.844190] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_vif_linux_bridge.forward_bridge_interface = ['all'] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.844390] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_vif_linux_bridge.iptables_bottom_regex = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.844575] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_vif_linux_bridge.iptables_drop_action = DROP {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.844749] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_vif_linux_bridge.iptables_top_regex = {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.844917] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_vif_linux_bridge.network_device_mtu = 1500 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.845098] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_vif_linux_bridge.use_ipv6 = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.845265] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_vif_linux_bridge.vlan_interface = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.845455] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_vif_ovs.default_qos_type = linux-noop {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.845629] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_vif_ovs.isolate_vif = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.845798] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_vif_ovs.network_device_mtu = 1500 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.846040] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_vif_ovs.ovs_vsctl_timeout = 120 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.846253] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_vif_ovs.ovsdb_connection = tcp:127.0.0.1:6640 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.846459] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_vif_ovs.ovsdb_interface = native {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.846636] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] os_vif_ovs.per_port_bridge = False {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.846814] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] privsep_osbrick.capabilities = [21] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.846980] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] privsep_osbrick.group = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.847148] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] privsep_osbrick.helper_command = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.847317] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] privsep_osbrick.logger_name = os_brick.privileged {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.847484] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] privsep_osbrick.thread_pool_size = 8 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.847642] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] privsep_osbrick.user = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.847815] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] nova_sys_admin.capabilities = [0, 1, 2, 3, 12, 21] {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.847974] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] nova_sys_admin.group = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.848148] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] nova_sys_admin.helper_command = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.848318] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] nova_sys_admin.logger_name = oslo_privsep.daemon {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.848486] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] nova_sys_admin.thread_pool_size = 8 {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.848661] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] nova_sys_admin.user = None {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 488.848800] env[62499]: DEBUG oslo_service.service [None req-11100c6b-1d61-45d0-9291-05646747c0ca None None] ******************************************************************************** {{(pid=62499) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2830}} [ 488.849308] env[62499]: INFO nova.service [-] Starting compute node (version 0.0.1) [ 489.352228] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Getting list of instances from cluster (obj){ [ 489.352228] env[62499]: value = "domain-c8" [ 489.352228] env[62499]: _type = "ClusterComputeResource" [ 489.352228] env[62499]: } {{(pid=62499) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2116}} [ 489.353395] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-07b406b5-68c8-4dbc-b957-e7f292093fa2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 489.362219] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Got total of 0 instances {{(pid=62499) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2125}} [ 489.362780] env[62499]: WARNING nova.virt.vmwareapi.driver [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] The vmwareapi driver is not tested by the OpenStack project nor does it have clear maintainer(s) and thus its quality can not be ensured. It should be considered experimental and may be removed in a future release. If you are using the driver in production please let us know via the openstack-discuss mailing list. [ 489.363299] env[62499]: INFO nova.virt.node [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Generated node identity 38a6c941-9755-44af-8da8-6ef6281d0157 [ 489.363525] env[62499]: INFO nova.virt.node [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Wrote node identity 38a6c941-9755-44af-8da8-6ef6281d0157 to /opt/stack/data/n-cpu-1/compute_id [ 489.865929] env[62499]: WARNING nova.compute.manager [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Compute nodes ['38a6c941-9755-44af-8da8-6ef6281d0157'] for host cpu-1 were not found in the database. If this is the first time this service is starting on this host, then you can ignore this warning. [ 490.870977] env[62499]: INFO nova.compute.manager [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Looking for unclaimed instances stuck in BUILDING status for nodes managed by this host [ 491.877119] env[62499]: WARNING nova.compute.manager [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] No compute node record found for host cpu-1. If this is the first time this service is starting on this host, then you can ignore this warning.: nova.exception_Remote.ComputeHostNotFound_Remote: Compute host cpu-1 could not be found. [ 491.877506] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 491.877634] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 491.877787] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 491.877939] env[62499]: DEBUG nova.compute.resource_tracker [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62499) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 491.878872] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1f65293-524c-4931-9e5f-b09a25783487 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 491.887111] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3db81b49-ad61-4b67-9479-d7a02a7ea2c1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 491.900198] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d08c9d8-4306-4885-8b93-3bb2f0262458 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 491.906425] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21c505da-4e89-4ac0-a07c-01169e1a3ba4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 491.935302] env[62499]: DEBUG nova.compute.resource_tracker [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181530MB free_disk=131GB free_vcpus=48 pci_devices=None {{(pid=62499) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 491.935482] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 491.935641] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 492.438552] env[62499]: WARNING nova.compute.resource_tracker [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] No compute node record for cpu-1:38a6c941-9755-44af-8da8-6ef6281d0157: nova.exception_Remote.ComputeHostNotFound_Remote: Compute host 38a6c941-9755-44af-8da8-6ef6281d0157 could not be found. [ 492.942175] env[62499]: INFO nova.compute.resource_tracker [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Compute node record created for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 with uuid: 38a6c941-9755-44af-8da8-6ef6281d0157 [ 494.450256] env[62499]: DEBUG nova.compute.resource_tracker [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62499) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 494.450639] env[62499]: DEBUG nova.compute.resource_tracker [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62499) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 494.601018] env[62499]: INFO nova.scheduler.client.report [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] [req-071be0cc-3d72-480c-a91c-dbf49192336d] Created resource provider record via placement API for resource provider with UUID 38a6c941-9755-44af-8da8-6ef6281d0157 and name domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28. [ 494.617573] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aacfa82a-072f-4316-b37a-7c00f78c18a5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 494.625018] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97f761dc-874a-432f-a8f2-b268629a81fb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 494.653883] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a266cdaf-3bd7-4971-9d39-7296a156c7b9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 494.660895] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ecf1709a-0ede-4ece-87c2-6c5c9f9eb29c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 494.673311] env[62499]: DEBUG nova.compute.provider_tree [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 495.207032] env[62499]: DEBUG nova.scheduler.client.report [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Updated inventory for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with generation 0 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:971}} [ 495.207284] env[62499]: DEBUG nova.compute.provider_tree [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Updating resource provider 38a6c941-9755-44af-8da8-6ef6281d0157 generation from 0 to 1 during operation: update_inventory {{(pid=62499) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 495.207435] env[62499]: DEBUG nova.compute.provider_tree [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 495.257968] env[62499]: DEBUG nova.compute.provider_tree [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Updating resource provider 38a6c941-9755-44af-8da8-6ef6281d0157 generation from 1 to 2 during operation: update_traits {{(pid=62499) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 495.765308] env[62499]: DEBUG nova.compute.resource_tracker [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62499) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 495.765720] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 3.830s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 495.765720] env[62499]: DEBUG nova.service [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Creating RPC server for service compute {{(pid=62499) start /opt/stack/nova/nova/service.py:186}} [ 495.778715] env[62499]: DEBUG nova.service [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] Join ServiceGroup membership for this service compute {{(pid=62499) start /opt/stack/nova/nova/service.py:203}} [ 495.778994] env[62499]: DEBUG nova.servicegroup.drivers.db [None req-4fd915f0-62d4-4723-a440-aedd1295fcb4 None None] DB_Driver: join new ServiceGroup member cpu-1 to the compute group, service = {{(pid=62499) join /opt/stack/nova/nova/servicegroup/drivers/db.py:44}} [ 518.784666] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._sync_power_states {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 519.290026] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Getting list of instances from cluster (obj){ [ 519.290026] env[62499]: value = "domain-c8" [ 519.290026] env[62499]: _type = "ClusterComputeResource" [ 519.290026] env[62499]: } {{(pid=62499) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2116}} [ 519.290026] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-275352b6-2f8a-4b59-ac51-f1b604fc6568 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 519.297059] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Got total of 0 instances {{(pid=62499) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2125}} [ 519.297490] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._cleanup_running_deleted_instances {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 519.297901] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Getting list of instances from cluster (obj){ [ 519.297901] env[62499]: value = "domain-c8" [ 519.297901] env[62499]: _type = "ClusterComputeResource" [ 519.297901] env[62499]: } {{(pid=62499) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2116}} [ 519.298926] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-41b62e8b-4cf7-4b09-b816-ce2ff7f4a068 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 519.307226] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Got total of 0 instances {{(pid=62499) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2125}} [ 532.134674] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Acquiring lock "ebd5745d-f073-429c-b609-a7520a539f2b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 532.134962] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Lock "ebd5745d-f073-429c-b609-a7520a539f2b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 532.639462] env[62499]: DEBUG nova.compute.manager [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 532.938827] env[62499]: DEBUG oslo_concurrency.lockutils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Acquiring lock "22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 532.939387] env[62499]: DEBUG oslo_concurrency.lockutils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Lock "22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 533.183700] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 533.184181] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 533.186838] env[62499]: INFO nova.compute.claims [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 533.441576] env[62499]: DEBUG nova.compute.manager [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 533.970796] env[62499]: DEBUG oslo_concurrency.lockutils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 534.260231] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa30691f-8d6e-4835-aa62-2527c3525a23 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.268105] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-53fe763a-3920-4b01-a7c3-64487f1387b6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.298518] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a053c5c1-fb25-41f5-99b6-c5dc9907429d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.306863] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59708fe4-4e08-47f7-9f4e-6cec8123b4c8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.325020] env[62499]: DEBUG nova.compute.provider_tree [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 534.523512] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Acquiring lock "904c5d79-86f3-4d5b-b8ba-7e3d515b32c0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 534.524251] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Lock "904c5d79-86f3-4d5b-b8ba-7e3d515b32c0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 534.828397] env[62499]: DEBUG nova.scheduler.client.report [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 535.027574] env[62499]: DEBUG nova.compute.manager [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 535.258930] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquiring lock "ddd212aa-9f4a-4b9d-aa60-cd4b64580f84" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.259232] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Lock "ddd212aa-9f4a-4b9d-aa60-cd4b64580f84" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.005s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.334923] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.151s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.335463] env[62499]: DEBUG nova.compute.manager [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 535.337922] env[62499]: DEBUG oslo_concurrency.lockutils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 1.367s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.343019] env[62499]: INFO nova.compute.claims [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 535.555934] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.762614] env[62499]: DEBUG nova.compute.manager [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 535.838789] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Acquiring lock "812c740f-eae7-4f6d-9e0d-8c16a4685a50" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.839053] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Lock "812c740f-eae7-4f6d-9e0d-8c16a4685a50" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.844726] env[62499]: DEBUG nova.compute.utils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 535.848063] env[62499]: DEBUG nova.compute.manager [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 535.848240] env[62499]: DEBUG nova.network.neutron [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 536.285024] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 536.343211] env[62499]: DEBUG nova.compute.manager [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 536.357185] env[62499]: DEBUG nova.compute.manager [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 536.411950] env[62499]: DEBUG nova.policy [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'fd02702ca4f740608114918e97833c16', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'a182eb3e4cc24706883605441eac740d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 536.485176] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a5aadacb-2f89-47a2-b5d1-f70b0b245a64 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 536.493026] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a46d42d5-10ab-489a-9070-ca5e025ee49f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 536.524604] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e5a1c24-7e89-4cbf-81c3-ab360e9006e9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 536.533126] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3c7cd54-1db7-4f25-bbbe-ec485f8e61e0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 536.547110] env[62499]: DEBUG nova.compute.provider_tree [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 536.684525] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Acquiring lock "48fc0cbf-18a6-4bbc-8455-93800b911d6a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 536.684578] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Lock "48fc0cbf-18a6-4bbc-8455-93800b911d6a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 536.878172] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 537.050931] env[62499]: DEBUG nova.scheduler.client.report [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 537.133997] env[62499]: DEBUG nova.network.neutron [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Successfully created port: d19aae75-6444-4c54-9764-32b1260ecf19 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 537.192393] env[62499]: DEBUG nova.compute.manager [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 537.374285] env[62499]: DEBUG nova.compute.manager [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 537.405554] env[62499]: DEBUG nova.virt.hardware [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 537.405797] env[62499]: DEBUG nova.virt.hardware [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 537.405949] env[62499]: DEBUG nova.virt.hardware [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 537.407019] env[62499]: DEBUG nova.virt.hardware [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 537.407019] env[62499]: DEBUG nova.virt.hardware [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 537.407137] env[62499]: DEBUG nova.virt.hardware [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 537.407443] env[62499]: DEBUG nova.virt.hardware [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 537.407672] env[62499]: DEBUG nova.virt.hardware [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 537.408497] env[62499]: DEBUG nova.virt.hardware [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 537.408935] env[62499]: DEBUG nova.virt.hardware [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 537.409022] env[62499]: DEBUG nova.virt.hardware [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 537.410199] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e79d17e-165b-4a69-8294-a19121b43690 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 537.423862] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5ba03ef-1d13-4ba9-ab0a-aca8abf49a19 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 537.444835] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-192d0ef3-5014-4393-908a-c2b5a51e4c5b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 537.556680] env[62499]: DEBUG oslo_concurrency.lockutils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.218s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 537.557133] env[62499]: DEBUG nova.compute.manager [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 537.563146] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 2.007s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 537.565574] env[62499]: INFO nova.compute.claims [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 537.732699] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 538.072433] env[62499]: DEBUG nova.compute.utils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 538.076430] env[62499]: DEBUG nova.compute.manager [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 538.076696] env[62499]: DEBUG nova.network.neutron [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 538.151759] env[62499]: DEBUG nova.policy [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '0b7a1abad9e64791b4229f67d838f8b1', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e4f68b27809f4bb58495a32dae15aa55', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 538.587946] env[62499]: DEBUG nova.compute.manager [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 538.745466] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d79a1ef-640f-465a-b608-ea9d9d97d600 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 538.754214] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c17af7b-d800-4f55-969f-deae01d09c44 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 538.789760] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c678ee9e-8f13-4e43-8774-90b1c96e66c8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 538.797986] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aea7a8cd-607f-491e-85f7-6c916b808003 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 538.812243] env[62499]: DEBUG nova.compute.provider_tree [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 538.838771] env[62499]: DEBUG oslo_concurrency.lockutils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Acquiring lock "99f77fa6-a625-4771-83c9-012f75ed1c7d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 538.839734] env[62499]: DEBUG oslo_concurrency.lockutils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Lock "99f77fa6-a625-4771-83c9-012f75ed1c7d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 538.895430] env[62499]: DEBUG nova.network.neutron [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Successfully created port: 8fd4fe87-772a-4544-b201-6fed68efdb0a {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 539.316053] env[62499]: DEBUG nova.scheduler.client.report [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 539.343757] env[62499]: DEBUG nova.compute.manager [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 539.599658] env[62499]: DEBUG nova.compute.manager [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 539.647319] env[62499]: DEBUG nova.virt.hardware [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 539.647595] env[62499]: DEBUG nova.virt.hardware [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 539.654772] env[62499]: DEBUG nova.virt.hardware [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 539.654772] env[62499]: DEBUG nova.virt.hardware [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 539.654772] env[62499]: DEBUG nova.virt.hardware [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 539.654772] env[62499]: DEBUG nova.virt.hardware [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 539.654772] env[62499]: DEBUG nova.virt.hardware [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 539.655237] env[62499]: DEBUG nova.virt.hardware [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 539.655237] env[62499]: DEBUG nova.virt.hardware [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 539.655237] env[62499]: DEBUG nova.virt.hardware [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 539.655237] env[62499]: DEBUG nova.virt.hardware [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 539.655883] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-efdb2d21-3fbb-4072-8d9e-b50e23f9525c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 539.671149] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea81d810-5876-4bef-8632-7081ad042092 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 539.822270] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.259s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 539.822459] env[62499]: DEBUG nova.compute.manager [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 539.825081] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 3.540s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 539.826419] env[62499]: INFO nova.compute.claims [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 539.885233] env[62499]: DEBUG oslo_concurrency.lockutils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 540.335049] env[62499]: DEBUG nova.compute.utils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 540.335049] env[62499]: DEBUG nova.compute.manager [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 540.335049] env[62499]: DEBUG nova.network.neutron [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 540.514873] env[62499]: ERROR nova.compute.manager [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d19aae75-6444-4c54-9764-32b1260ecf19, please check neutron logs for more information. [ 540.514873] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 540.514873] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 540.514873] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 540.514873] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 540.514873] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 540.514873] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 540.514873] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 540.514873] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 540.514873] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 540.514873] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 540.514873] env[62499]: ERROR nova.compute.manager raise self.value [ 540.514873] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 540.514873] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 540.514873] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 540.514873] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 540.515372] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 540.515372] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 540.515372] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d19aae75-6444-4c54-9764-32b1260ecf19, please check neutron logs for more information. [ 540.515372] env[62499]: ERROR nova.compute.manager [ 540.515912] env[62499]: Traceback (most recent call last): [ 540.516251] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 540.516251] env[62499]: listener.cb(fileno) [ 540.519026] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 540.519026] env[62499]: result = function(*args, **kwargs) [ 540.519026] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 540.519026] env[62499]: return func(*args, **kwargs) [ 540.519026] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 540.519026] env[62499]: raise e [ 540.519026] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 540.519026] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 540.519026] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 540.519026] env[62499]: created_port_ids = self._update_ports_for_instance( [ 540.519026] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 540.519026] env[62499]: with excutils.save_and_reraise_exception(): [ 540.519026] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 540.519026] env[62499]: self.force_reraise() [ 540.519026] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 540.519026] env[62499]: raise self.value [ 540.519026] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 540.519026] env[62499]: updated_port = self._update_port( [ 540.519026] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 540.519026] env[62499]: _ensure_no_port_binding_failure(port) [ 540.519026] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 540.519026] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 540.519026] env[62499]: nova.exception.PortBindingFailed: Binding failed for port d19aae75-6444-4c54-9764-32b1260ecf19, please check neutron logs for more information. [ 540.519026] env[62499]: Removing descriptor: 15 [ 540.519026] env[62499]: ERROR nova.compute.manager [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d19aae75-6444-4c54-9764-32b1260ecf19, please check neutron logs for more information. [ 540.519880] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Traceback (most recent call last): [ 540.519880] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 540.519880] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] yield resources [ 540.519880] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 540.519880] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] self.driver.spawn(context, instance, image_meta, [ 540.519880] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 540.519880] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 540.519880] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 540.519880] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] vm_ref = self.build_virtual_machine(instance, [ 540.519880] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 540.519880] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] vif_infos = vmwarevif.get_vif_info(self._session, [ 540.519880] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 540.519880] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] for vif in network_info: [ 540.520266] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 540.520266] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] return self._sync_wrapper(fn, *args, **kwargs) [ 540.520266] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 540.520266] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] self.wait() [ 540.520266] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 540.520266] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] self[:] = self._gt.wait() [ 540.520266] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 540.520266] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] return self._exit_event.wait() [ 540.520266] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 540.520266] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] result = hub.switch() [ 540.520266] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 540.520266] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] return self.greenlet.switch() [ 540.520266] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 540.520581] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] result = function(*args, **kwargs) [ 540.520581] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 540.520581] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] return func(*args, **kwargs) [ 540.520581] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 540.520581] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] raise e [ 540.520581] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 540.520581] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] nwinfo = self.network_api.allocate_for_instance( [ 540.520581] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 540.520581] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] created_port_ids = self._update_ports_for_instance( [ 540.520581] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 540.520581] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] with excutils.save_and_reraise_exception(): [ 540.520581] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 540.520581] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] self.force_reraise() [ 540.520926] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 540.520926] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] raise self.value [ 540.520926] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 540.520926] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] updated_port = self._update_port( [ 540.520926] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 540.520926] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] _ensure_no_port_binding_failure(port) [ 540.520926] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 540.520926] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] raise exception.PortBindingFailed(port_id=port['id']) [ 540.520926] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] nova.exception.PortBindingFailed: Binding failed for port d19aae75-6444-4c54-9764-32b1260ecf19, please check neutron logs for more information. [ 540.520926] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] [ 540.520926] env[62499]: INFO nova.compute.manager [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Terminating instance [ 540.524848] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Acquiring lock "refresh_cache-ebd5745d-f073-429c-b609-a7520a539f2b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 540.524848] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Acquired lock "refresh_cache-ebd5745d-f073-429c-b609-a7520a539f2b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 540.524848] env[62499]: DEBUG nova.network.neutron [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 540.527574] env[62499]: DEBUG nova.policy [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f05975529be14a03b5f165f134a770d4', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'a5a2a6c3b01640eba0a5b5612711385d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 540.840122] env[62499]: DEBUG nova.compute.manager [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 540.982420] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f254396b-5733-4fff-835b-aa64c5599392 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 540.990950] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d6eb1a75-bee4-4209-bafa-6fd63612dc86 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 541.021802] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3420ff5c-3898-4db2-aec0-f23a083d1900 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 541.030663] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d6c893c3-45e7-4e44-9a45-0bcd2c1a5bea {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 541.050331] env[62499]: DEBUG nova.compute.provider_tree [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 541.080099] env[62499]: DEBUG nova.network.neutron [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 541.169104] env[62499]: DEBUG nova.compute.manager [req-831cc7d9-6d34-41c2-8f9e-7bdc07456a04 req-51140415-3387-4de9-85ed-309ecd5282e7 service nova] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Received event network-changed-d19aae75-6444-4c54-9764-32b1260ecf19 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 541.169310] env[62499]: DEBUG nova.compute.manager [req-831cc7d9-6d34-41c2-8f9e-7bdc07456a04 req-51140415-3387-4de9-85ed-309ecd5282e7 service nova] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Refreshing instance network info cache due to event network-changed-d19aae75-6444-4c54-9764-32b1260ecf19. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 541.169878] env[62499]: DEBUG oslo_concurrency.lockutils [req-831cc7d9-6d34-41c2-8f9e-7bdc07456a04 req-51140415-3387-4de9-85ed-309ecd5282e7 service nova] Acquiring lock "refresh_cache-ebd5745d-f073-429c-b609-a7520a539f2b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 541.204931] env[62499]: DEBUG nova.network.neutron [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 541.553497] env[62499]: DEBUG nova.scheduler.client.report [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 541.689729] env[62499]: DEBUG nova.network.neutron [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Successfully created port: d253c0f7-c107-4caf-8d19-68e503db563f {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 541.708569] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Releasing lock "refresh_cache-ebd5745d-f073-429c-b609-a7520a539f2b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 541.709301] env[62499]: DEBUG nova.compute.manager [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 541.710031] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 541.710031] env[62499]: DEBUG oslo_concurrency.lockutils [req-831cc7d9-6d34-41c2-8f9e-7bdc07456a04 req-51140415-3387-4de9-85ed-309ecd5282e7 service nova] Acquired lock "refresh_cache-ebd5745d-f073-429c-b609-a7520a539f2b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 541.710323] env[62499]: DEBUG nova.network.neutron [req-831cc7d9-6d34-41c2-8f9e-7bdc07456a04 req-51140415-3387-4de9-85ed-309ecd5282e7 service nova] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Refreshing network info cache for port d19aae75-6444-4c54-9764-32b1260ecf19 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 541.714088] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-0b513d24-7e10-4319-aeda-1d8662697853 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 541.724949] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2764d569-ea92-48d2-ae75-52be9bb6b814 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 541.751208] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ebd5745d-f073-429c-b609-a7520a539f2b could not be found. [ 541.751448] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 541.751843] env[62499]: INFO nova.compute.manager [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Took 0.04 seconds to destroy the instance on the hypervisor. [ 541.752171] env[62499]: DEBUG oslo.service.loopingcall [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 541.752915] env[62499]: DEBUG nova.compute.manager [-] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 541.752915] env[62499]: DEBUG nova.network.neutron [-] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 541.766043] env[62499]: ERROR nova.compute.manager [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 8fd4fe87-772a-4544-b201-6fed68efdb0a, please check neutron logs for more information. [ 541.766043] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 541.766043] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 541.766043] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 541.766043] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 541.766043] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 541.766043] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 541.766043] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 541.766043] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 541.766043] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 541.766043] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 541.766043] env[62499]: ERROR nova.compute.manager raise self.value [ 541.766043] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 541.766043] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 541.766043] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 541.766043] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 541.767029] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 541.767029] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 541.767029] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 8fd4fe87-772a-4544-b201-6fed68efdb0a, please check neutron logs for more information. [ 541.767029] env[62499]: ERROR nova.compute.manager [ 541.767029] env[62499]: Traceback (most recent call last): [ 541.767029] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 541.767029] env[62499]: listener.cb(fileno) [ 541.767029] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 541.767029] env[62499]: result = function(*args, **kwargs) [ 541.767029] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 541.767029] env[62499]: return func(*args, **kwargs) [ 541.767029] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 541.767029] env[62499]: raise e [ 541.767029] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 541.767029] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 541.767029] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 541.767029] env[62499]: created_port_ids = self._update_ports_for_instance( [ 541.767029] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 541.767029] env[62499]: with excutils.save_and_reraise_exception(): [ 541.767029] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 541.767029] env[62499]: self.force_reraise() [ 541.767029] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 541.767029] env[62499]: raise self.value [ 541.767029] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 541.767029] env[62499]: updated_port = self._update_port( [ 541.767029] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 541.767029] env[62499]: _ensure_no_port_binding_failure(port) [ 541.767029] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 541.767029] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 541.767833] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 8fd4fe87-772a-4544-b201-6fed68efdb0a, please check neutron logs for more information. [ 541.767833] env[62499]: Removing descriptor: 16 [ 541.767833] env[62499]: ERROR nova.compute.manager [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 8fd4fe87-772a-4544-b201-6fed68efdb0a, please check neutron logs for more information. [ 541.767833] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Traceback (most recent call last): [ 541.767833] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 541.767833] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] yield resources [ 541.767833] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 541.767833] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] self.driver.spawn(context, instance, image_meta, [ 541.767833] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 541.767833] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] self._vmops.spawn(context, instance, image_meta, injected_files, [ 541.767833] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 541.767833] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] vm_ref = self.build_virtual_machine(instance, [ 541.768185] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 541.768185] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] vif_infos = vmwarevif.get_vif_info(self._session, [ 541.768185] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 541.768185] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] for vif in network_info: [ 541.768185] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 541.768185] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] return self._sync_wrapper(fn, *args, **kwargs) [ 541.768185] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 541.768185] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] self.wait() [ 541.768185] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 541.768185] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] self[:] = self._gt.wait() [ 541.768185] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 541.768185] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] return self._exit_event.wait() [ 541.768185] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 541.768548] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] result = hub.switch() [ 541.768548] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 541.768548] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] return self.greenlet.switch() [ 541.768548] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 541.768548] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] result = function(*args, **kwargs) [ 541.768548] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 541.768548] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] return func(*args, **kwargs) [ 541.768548] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 541.768548] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] raise e [ 541.768548] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 541.768548] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] nwinfo = self.network_api.allocate_for_instance( [ 541.768548] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 541.768548] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] created_port_ids = self._update_ports_for_instance( [ 541.768911] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 541.768911] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] with excutils.save_and_reraise_exception(): [ 541.768911] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 541.768911] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] self.force_reraise() [ 541.768911] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 541.768911] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] raise self.value [ 541.768911] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 541.768911] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] updated_port = self._update_port( [ 541.768911] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 541.768911] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] _ensure_no_port_binding_failure(port) [ 541.768911] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 541.768911] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] raise exception.PortBindingFailed(port_id=port['id']) [ 541.769274] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] nova.exception.PortBindingFailed: Binding failed for port 8fd4fe87-772a-4544-b201-6fed68efdb0a, please check neutron logs for more information. [ 541.769274] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] [ 541.769274] env[62499]: INFO nova.compute.manager [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Terminating instance [ 541.770957] env[62499]: DEBUG oslo_concurrency.lockutils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Acquiring lock "refresh_cache-22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 541.771020] env[62499]: DEBUG oslo_concurrency.lockutils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Acquired lock "refresh_cache-22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 541.772590] env[62499]: DEBUG nova.network.neutron [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 541.783901] env[62499]: DEBUG nova.network.neutron [-] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 541.861309] env[62499]: DEBUG nova.compute.manager [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 541.900416] env[62499]: DEBUG nova.virt.hardware [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 541.900730] env[62499]: DEBUG nova.virt.hardware [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 541.900819] env[62499]: DEBUG nova.virt.hardware [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 541.901021] env[62499]: DEBUG nova.virt.hardware [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 541.901162] env[62499]: DEBUG nova.virt.hardware [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 541.901308] env[62499]: DEBUG nova.virt.hardware [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 541.901510] env[62499]: DEBUG nova.virt.hardware [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 541.901720] env[62499]: DEBUG nova.virt.hardware [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 541.901839] env[62499]: DEBUG nova.virt.hardware [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 541.901978] env[62499]: DEBUG nova.virt.hardware [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 541.903114] env[62499]: DEBUG nova.virt.hardware [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 541.903114] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8fe29771-efce-40f9-8aa6-e335ace1cb3a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 541.913778] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9dbd66af-58b8-4b92-b79a-fa355fb9242e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 542.061514] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.236s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 542.065246] env[62499]: DEBUG nova.compute.manager [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 542.070919] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 5.192s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 542.072016] env[62499]: INFO nova.compute.claims [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 542.287822] env[62499]: DEBUG nova.network.neutron [-] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 542.351337] env[62499]: DEBUG nova.network.neutron [req-831cc7d9-6d34-41c2-8f9e-7bdc07456a04 req-51140415-3387-4de9-85ed-309ecd5282e7 service nova] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 542.368780] env[62499]: DEBUG nova.network.neutron [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 542.570039] env[62499]: DEBUG nova.network.neutron [req-831cc7d9-6d34-41c2-8f9e-7bdc07456a04 req-51140415-3387-4de9-85ed-309ecd5282e7 service nova] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 542.582445] env[62499]: DEBUG nova.compute.utils [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 542.587253] env[62499]: DEBUG nova.compute.manager [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Not allocating networking since 'none' was specified. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 542.663594] env[62499]: DEBUG nova.network.neutron [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 542.791458] env[62499]: INFO nova.compute.manager [-] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Took 1.04 seconds to deallocate network for instance. [ 542.794776] env[62499]: DEBUG nova.compute.claims [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 542.794976] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 543.078190] env[62499]: DEBUG oslo_concurrency.lockutils [req-831cc7d9-6d34-41c2-8f9e-7bdc07456a04 req-51140415-3387-4de9-85ed-309ecd5282e7 service nova] Releasing lock "refresh_cache-ebd5745d-f073-429c-b609-a7520a539f2b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 543.090433] env[62499]: DEBUG nova.compute.manager [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 543.167383] env[62499]: DEBUG oslo_concurrency.lockutils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Releasing lock "refresh_cache-22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 543.167789] env[62499]: DEBUG nova.compute.manager [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 543.167975] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 543.168904] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-053eac03-9606-4d04-a6e1-c7eb91c1d434 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 543.178781] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f044f118-7f82-4814-bf6c-00eda39fee2b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 543.211644] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa could not be found. [ 543.212129] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 543.212129] env[62499]: INFO nova.compute.manager [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Took 0.04 seconds to destroy the instance on the hypervisor. [ 543.212337] env[62499]: DEBUG oslo.service.loopingcall [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 543.212923] env[62499]: DEBUG nova.compute.manager [-] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 543.212923] env[62499]: DEBUG nova.network.neutron [-] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 543.264248] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed303d0d-724a-477c-afbd-04a69af71315 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 543.277426] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67ba0a8c-405f-4065-a669-0672d04ffe48 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 543.317388] env[62499]: DEBUG nova.network.neutron [-] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 543.319031] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec6bc488-0b04-451d-b15f-7f22ad38f6ef {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 543.327052] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c3f94ec-3132-4f73-996d-1ce82b050af7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 543.342796] env[62499]: DEBUG nova.compute.provider_tree [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 543.825148] env[62499]: DEBUG nova.network.neutron [-] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 543.847845] env[62499]: DEBUG nova.scheduler.client.report [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 544.055850] env[62499]: DEBUG nova.compute.manager [req-d3c62e41-b78f-4f33-bff6-6e53238719b0 req-e7ac1035-857f-45d8-8b8d-181587d37780 service nova] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Received event network-vif-deleted-d19aae75-6444-4c54-9764-32b1260ecf19 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 544.056151] env[62499]: DEBUG nova.compute.manager [req-d3c62e41-b78f-4f33-bff6-6e53238719b0 req-e7ac1035-857f-45d8-8b8d-181587d37780 service nova] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Received event network-changed-8fd4fe87-772a-4544-b201-6fed68efdb0a {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 544.056262] env[62499]: DEBUG nova.compute.manager [req-d3c62e41-b78f-4f33-bff6-6e53238719b0 req-e7ac1035-857f-45d8-8b8d-181587d37780 service nova] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Refreshing instance network info cache due to event network-changed-8fd4fe87-772a-4544-b201-6fed68efdb0a. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 544.056706] env[62499]: DEBUG oslo_concurrency.lockutils [req-d3c62e41-b78f-4f33-bff6-6e53238719b0 req-e7ac1035-857f-45d8-8b8d-181587d37780 service nova] Acquiring lock "refresh_cache-22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 544.056706] env[62499]: DEBUG oslo_concurrency.lockutils [req-d3c62e41-b78f-4f33-bff6-6e53238719b0 req-e7ac1035-857f-45d8-8b8d-181587d37780 service nova] Acquired lock "refresh_cache-22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 544.056826] env[62499]: DEBUG nova.network.neutron [req-d3c62e41-b78f-4f33-bff6-6e53238719b0 req-e7ac1035-857f-45d8-8b8d-181587d37780 service nova] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Refreshing network info cache for port 8fd4fe87-772a-4544-b201-6fed68efdb0a {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 544.103026] env[62499]: DEBUG nova.compute.manager [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 544.137990] env[62499]: DEBUG nova.virt.hardware [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 544.137990] env[62499]: DEBUG nova.virt.hardware [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 544.137990] env[62499]: DEBUG nova.virt.hardware [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 544.137990] env[62499]: DEBUG nova.virt.hardware [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 544.138326] env[62499]: DEBUG nova.virt.hardware [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 544.138326] env[62499]: DEBUG nova.virt.hardware [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 544.138326] env[62499]: DEBUG nova.virt.hardware [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 544.138326] env[62499]: DEBUG nova.virt.hardware [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 544.138437] env[62499]: DEBUG nova.virt.hardware [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 544.138572] env[62499]: DEBUG nova.virt.hardware [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 544.138744] env[62499]: DEBUG nova.virt.hardware [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 544.140149] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-62beac2f-8f96-48db-9dd3-203431eea5e5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 544.154057] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-213564db-d4b7-4acf-81b8-eb12f01affd2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 544.171729] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Instance VIF info [] {{(pid=62499) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 544.184077] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Creating folder: OpenStack. Parent ref: group-v4. {{(pid=62499) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 544.184077] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-577b5dc7-6924-45f8-a9ab-663a3386eb4c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 544.196593] env[62499]: INFO nova.virt.vmwareapi.vm_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Created folder: OpenStack in parent group-v4. [ 544.196803] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Creating folder: Project (27c9edfccbc6413ba8f969292695b2c7). Parent ref: group-v285191. {{(pid=62499) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 544.197088] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-55684999-2c9b-4f22-a8fa-b751f56ef58b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 544.209670] env[62499]: INFO nova.virt.vmwareapi.vm_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Created folder: Project (27c9edfccbc6413ba8f969292695b2c7) in parent group-v285191. [ 544.209733] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Creating folder: Instances. Parent ref: group-v285192. {{(pid=62499) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 544.209955] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-9d81387c-35c1-4145-8e5f-7e26e75ac6fb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 544.220618] env[62499]: INFO nova.virt.vmwareapi.vm_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Created folder: Instances in parent group-v285192. [ 544.220618] env[62499]: DEBUG oslo.service.loopingcall [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 544.220618] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Creating VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 544.220618] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-58ae72bb-e348-4fd7-ba85-88b2fc45ef82 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 544.240822] env[62499]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 544.240822] env[62499]: value = "task-1335640" [ 544.240822] env[62499]: _type = "Task" [ 544.240822] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 544.252409] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335640, 'name': CreateVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 544.285924] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 544.285924] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 544.286782] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Starting heal instance info cache {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9928}} [ 544.286782] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Rebuilding the list of instances to heal {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 544.327514] env[62499]: INFO nova.compute.manager [-] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Took 1.11 seconds to deallocate network for instance. [ 544.331777] env[62499]: DEBUG nova.compute.claims [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 544.331777] env[62499]: DEBUG oslo_concurrency.lockutils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 544.354268] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.284s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 544.356621] env[62499]: DEBUG nova.compute.manager [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 544.362748] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 6.630s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 544.365883] env[62499]: INFO nova.compute.claims [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 544.630912] env[62499]: DEBUG nova.network.neutron [req-d3c62e41-b78f-4f33-bff6-6e53238719b0 req-e7ac1035-857f-45d8-8b8d-181587d37780 service nova] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 544.756936] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335640, 'name': CreateVM_Task} progress is 99%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 544.792847] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 544.793115] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 544.793782] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 544.794088] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 544.794350] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 544.794965] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Didn't find any instances for network info cache update. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10014}} [ 544.796330] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 544.796625] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 544.796842] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 544.797415] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 544.797415] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 544.797517] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 544.797642] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62499) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10547}} [ 544.797784] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager.update_available_resource {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 544.864226] env[62499]: DEBUG nova.compute.utils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 544.867983] env[62499]: DEBUG nova.compute.manager [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 544.868430] env[62499]: DEBUG nova.network.neutron [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 545.179853] env[62499]: DEBUG nova.network.neutron [req-d3c62e41-b78f-4f33-bff6-6e53238719b0 req-e7ac1035-857f-45d8-8b8d-181587d37780 service nova] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 545.254756] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335640, 'name': CreateVM_Task, 'duration_secs': 0.550913} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 545.255018] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Created VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 545.255952] env[62499]: DEBUG oslo_vmware.service [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99f0daeb-e7e7-40b8-907a-a01e73861e5b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 545.262929] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 545.263122] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquired lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 545.263797] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 545.264618] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-8b9ea8e7-9752-4e8c-8e79-3143ec53f1ab {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 545.267574] env[62499]: DEBUG nova.policy [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7f4cf53af57b4bae9ab5788f2d663dc1', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '26c92aca91614f018775d7e4949619d6', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 545.276361] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 545.276361] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]523fcf4c-e6fe-9412-5c7b-62afc1d710ee" [ 545.276361] env[62499]: _type = "Task" [ 545.276361] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 545.286133] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]523fcf4c-e6fe-9412-5c7b-62afc1d710ee, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 545.305274] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 545.372847] env[62499]: DEBUG nova.compute.manager [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 545.573044] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1bec37d-0e92-4658-9fb1-15520b76f584 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 545.588936] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-558d0d5d-78f7-4beb-a9c9-4b4a592868f0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 545.634526] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0baaea05-5679-41af-aefa-a349f357abc8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 545.642161] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-486d37b8-d2a8-45e8-abde-6d588dd8e809 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 545.659191] env[62499]: DEBUG nova.compute.provider_tree [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 545.685508] env[62499]: DEBUG oslo_concurrency.lockutils [req-d3c62e41-b78f-4f33-bff6-6e53238719b0 req-e7ac1035-857f-45d8-8b8d-181587d37780 service nova] Releasing lock "refresh_cache-22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 545.685508] env[62499]: DEBUG nova.compute.manager [req-d3c62e41-b78f-4f33-bff6-6e53238719b0 req-e7ac1035-857f-45d8-8b8d-181587d37780 service nova] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Received event network-vif-deleted-8fd4fe87-772a-4544-b201-6fed68efdb0a {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 545.710450] env[62499]: ERROR nova.compute.manager [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d253c0f7-c107-4caf-8d19-68e503db563f, please check neutron logs for more information. [ 545.710450] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 545.710450] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 545.710450] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 545.710450] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 545.710450] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 545.710450] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 545.710450] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 545.710450] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 545.710450] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 545.710450] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 545.710450] env[62499]: ERROR nova.compute.manager raise self.value [ 545.710450] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 545.710450] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 545.710450] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 545.710450] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 545.712868] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 545.712868] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 545.712868] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d253c0f7-c107-4caf-8d19-68e503db563f, please check neutron logs for more information. [ 545.712868] env[62499]: ERROR nova.compute.manager [ 545.712868] env[62499]: Traceback (most recent call last): [ 545.712868] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 545.712868] env[62499]: listener.cb(fileno) [ 545.712868] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 545.712868] env[62499]: result = function(*args, **kwargs) [ 545.712868] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 545.712868] env[62499]: return func(*args, **kwargs) [ 545.712868] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 545.712868] env[62499]: raise e [ 545.712868] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 545.712868] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 545.712868] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 545.712868] env[62499]: created_port_ids = self._update_ports_for_instance( [ 545.712868] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 545.712868] env[62499]: with excutils.save_and_reraise_exception(): [ 545.712868] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 545.712868] env[62499]: self.force_reraise() [ 545.712868] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 545.712868] env[62499]: raise self.value [ 545.712868] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 545.712868] env[62499]: updated_port = self._update_port( [ 545.712868] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 545.712868] env[62499]: _ensure_no_port_binding_failure(port) [ 545.712868] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 545.712868] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 545.713884] env[62499]: nova.exception.PortBindingFailed: Binding failed for port d253c0f7-c107-4caf-8d19-68e503db563f, please check neutron logs for more information. [ 545.713884] env[62499]: Removing descriptor: 17 [ 545.713884] env[62499]: ERROR nova.compute.manager [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d253c0f7-c107-4caf-8d19-68e503db563f, please check neutron logs for more information. [ 545.713884] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Traceback (most recent call last): [ 545.713884] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 545.713884] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] yield resources [ 545.713884] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 545.713884] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] self.driver.spawn(context, instance, image_meta, [ 545.713884] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 545.713884] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 545.713884] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 545.713884] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] vm_ref = self.build_virtual_machine(instance, [ 545.714874] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 545.714874] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] vif_infos = vmwarevif.get_vif_info(self._session, [ 545.714874] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 545.714874] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] for vif in network_info: [ 545.714874] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 545.714874] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] return self._sync_wrapper(fn, *args, **kwargs) [ 545.714874] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 545.714874] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] self.wait() [ 545.714874] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 545.714874] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] self[:] = self._gt.wait() [ 545.714874] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 545.714874] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] return self._exit_event.wait() [ 545.714874] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 545.715471] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] result = hub.switch() [ 545.715471] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 545.715471] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] return self.greenlet.switch() [ 545.715471] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 545.715471] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] result = function(*args, **kwargs) [ 545.715471] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 545.715471] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] return func(*args, **kwargs) [ 545.715471] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 545.715471] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] raise e [ 545.715471] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 545.715471] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] nwinfo = self.network_api.allocate_for_instance( [ 545.715471] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 545.715471] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] created_port_ids = self._update_ports_for_instance( [ 545.716872] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 545.716872] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] with excutils.save_and_reraise_exception(): [ 545.716872] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 545.716872] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] self.force_reraise() [ 545.716872] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 545.716872] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] raise self.value [ 545.716872] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 545.716872] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] updated_port = self._update_port( [ 545.716872] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 545.716872] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] _ensure_no_port_binding_failure(port) [ 545.716872] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 545.716872] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] raise exception.PortBindingFailed(port_id=port['id']) [ 545.717226] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] nova.exception.PortBindingFailed: Binding failed for port d253c0f7-c107-4caf-8d19-68e503db563f, please check neutron logs for more information. [ 545.717226] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] [ 545.717226] env[62499]: INFO nova.compute.manager [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Terminating instance [ 545.717226] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Acquiring lock "refresh_cache-904c5d79-86f3-4d5b-b8ba-7e3d515b32c0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 545.717226] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Acquired lock "refresh_cache-904c5d79-86f3-4d5b-b8ba-7e3d515b32c0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 545.717226] env[62499]: DEBUG nova.network.neutron [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 545.791460] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Releasing lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 545.793344] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Processing image 0422123c-16ce-4c9d-b1c6-0281fe0e54ad {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 545.793344] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 545.793344] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquired lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 545.793344] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 545.793344] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-ff495989-361d-4b76-b635-1f6459956b5a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 545.810365] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 545.810546] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62499) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 545.811372] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d282878d-408e-42af-bca3-139e485ca312 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 545.820717] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-e82b3795-edf7-4713-be1c-ad655061afa9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 545.828690] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 545.828690] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52f5dfdb-ca3a-aacb-ab76-cd922bb4c6ec" [ 545.828690] env[62499]: _type = "Task" [ 545.828690] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 545.840363] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52f5dfdb-ca3a-aacb-ab76-cd922bb4c6ec, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 546.163359] env[62499]: DEBUG nova.scheduler.client.report [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 546.276826] env[62499]: DEBUG nova.network.neutron [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 546.321195] env[62499]: DEBUG nova.compute.manager [req-c92936c2-0762-4aeb-8c9d-ae76e693de8c req-cfde7e9e-4ddf-4ec7-b9b6-8f954b0bc358 service nova] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Received event network-changed-d253c0f7-c107-4caf-8d19-68e503db563f {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 546.321858] env[62499]: DEBUG nova.compute.manager [req-c92936c2-0762-4aeb-8c9d-ae76e693de8c req-cfde7e9e-4ddf-4ec7-b9b6-8f954b0bc358 service nova] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Refreshing instance network info cache due to event network-changed-d253c0f7-c107-4caf-8d19-68e503db563f. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 546.321858] env[62499]: DEBUG oslo_concurrency.lockutils [req-c92936c2-0762-4aeb-8c9d-ae76e693de8c req-cfde7e9e-4ddf-4ec7-b9b6-8f954b0bc358 service nova] Acquiring lock "refresh_cache-904c5d79-86f3-4d5b-b8ba-7e3d515b32c0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 546.341026] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Preparing fetch location {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 546.341311] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Creating directory with path [datastore2] vmware_temp/7eeef8c1-5ccb-4b80-9ac3-c1526fd140d5/0422123c-16ce-4c9d-b1c6-0281fe0e54ad {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 546.341591] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-f399e361-2353-4939-907c-8f9b33a8b5b5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 546.353948] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Created directory with path [datastore2] vmware_temp/7eeef8c1-5ccb-4b80-9ac3-c1526fd140d5/0422123c-16ce-4c9d-b1c6-0281fe0e54ad {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 546.354622] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Fetch image to [datastore2] vmware_temp/7eeef8c1-5ccb-4b80-9ac3-c1526fd140d5/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/tmp-sparse.vmdk {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 546.354622] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Downloading image file data 0422123c-16ce-4c9d-b1c6-0281fe0e54ad to [datastore2] vmware_temp/7eeef8c1-5ccb-4b80-9ac3-c1526fd140d5/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/tmp-sparse.vmdk on the data store datastore2 {{(pid=62499) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 546.355161] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50027766-98b3-46dc-b82e-24b163307f0e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 546.365786] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b9b89c63-bc06-4676-97fd-4708cb31429f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 546.375535] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6573aa48-2d00-4537-8ae3-2089979c8006 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 546.408555] env[62499]: DEBUG nova.compute.manager [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 546.412789] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1fc053a-1dfa-4fee-af62-e014ba6a3857 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 546.418908] env[62499]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-7dd1a95a-3593-446a-bff4-8f9d757873e0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 546.441113] env[62499]: DEBUG nova.virt.hardware [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 546.441368] env[62499]: DEBUG nova.virt.hardware [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 546.441524] env[62499]: DEBUG nova.virt.hardware [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 546.441706] env[62499]: DEBUG nova.virt.hardware [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 546.441853] env[62499]: DEBUG nova.virt.hardware [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 546.441995] env[62499]: DEBUG nova.virt.hardware [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 546.442487] env[62499]: DEBUG nova.virt.hardware [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 546.442740] env[62499]: DEBUG nova.virt.hardware [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 546.442869] env[62499]: DEBUG nova.virt.hardware [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 546.442975] env[62499]: DEBUG nova.virt.hardware [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 546.443162] env[62499]: DEBUG nova.virt.hardware [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 546.444033] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7101bfd-261d-46c9-9c9e-8043cc1afd4f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 546.449653] env[62499]: DEBUG nova.virt.vmwareapi.images [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Downloading image file data 0422123c-16ce-4c9d-b1c6-0281fe0e54ad to the data store datastore2 {{(pid=62499) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 546.453453] env[62499]: DEBUG nova.network.neutron [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 546.468088] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7a97192-1f91-4c8d-9706-3910013d672a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 546.535821] env[62499]: DEBUG oslo_vmware.rw_handles [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/7eeef8c1-5ccb-4b80-9ac3-c1526fd140d5/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=62499) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 546.608583] env[62499]: DEBUG nova.network.neutron [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Successfully created port: 900edb48-3a43-497c-9834-e3a23a708ba3 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 546.675442] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.306s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 546.675442] env[62499]: DEBUG nova.compute.manager [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 546.680627] env[62499]: DEBUG oslo_concurrency.lockutils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 6.798s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 546.683098] env[62499]: INFO nova.compute.claims [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 546.961873] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Releasing lock "refresh_cache-904c5d79-86f3-4d5b-b8ba-7e3d515b32c0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 546.961873] env[62499]: DEBUG nova.compute.manager [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 546.961873] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 546.961873] env[62499]: DEBUG oslo_concurrency.lockutils [req-c92936c2-0762-4aeb-8c9d-ae76e693de8c req-cfde7e9e-4ddf-4ec7-b9b6-8f954b0bc358 service nova] Acquired lock "refresh_cache-904c5d79-86f3-4d5b-b8ba-7e3d515b32c0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 546.961873] env[62499]: DEBUG nova.network.neutron [req-c92936c2-0762-4aeb-8c9d-ae76e693de8c req-cfde7e9e-4ddf-4ec7-b9b6-8f954b0bc358 service nova] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Refreshing network info cache for port d253c0f7-c107-4caf-8d19-68e503db563f {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 546.962390] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-35849afb-82a3-4721-a195-74c8258888bb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 546.975166] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac089d13-39f6-4fc5-a867-93c92ad70f69 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 547.007753] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0 could not be found. [ 547.008109] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 547.008356] env[62499]: INFO nova.compute.manager [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Took 0.05 seconds to destroy the instance on the hypervisor. [ 547.008648] env[62499]: DEBUG oslo.service.loopingcall [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 547.009236] env[62499]: DEBUG nova.compute.manager [-] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 547.009389] env[62499]: DEBUG nova.network.neutron [-] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 547.105263] env[62499]: DEBUG nova.network.neutron [-] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 547.193494] env[62499]: DEBUG nova.compute.utils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 547.198571] env[62499]: DEBUG nova.compute.manager [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 547.198761] env[62499]: DEBUG nova.network.neutron [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 547.234929] env[62499]: DEBUG oslo_vmware.rw_handles [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Completed reading data from the image iterator. {{(pid=62499) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 547.235412] env[62499]: DEBUG oslo_vmware.rw_handles [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/7eeef8c1-5ccb-4b80-9ac3-c1526fd140d5/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=62499) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 547.382435] env[62499]: DEBUG nova.policy [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '11251536a5af4def955e95158d82fb81', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '18249883469c4514a8300d99f32482a5', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 547.551798] env[62499]: DEBUG nova.network.neutron [req-c92936c2-0762-4aeb-8c9d-ae76e693de8c req-cfde7e9e-4ddf-4ec7-b9b6-8f954b0bc358 service nova] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 547.615206] env[62499]: DEBUG nova.network.neutron [-] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 547.703028] env[62499]: DEBUG nova.compute.manager [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 547.871021] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-509af404-ea51-48ea-85e2-ce44bae2aff7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 547.875868] env[62499]: DEBUG nova.network.neutron [req-c92936c2-0762-4aeb-8c9d-ae76e693de8c req-cfde7e9e-4ddf-4ec7-b9b6-8f954b0bc358 service nova] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 547.883995] env[62499]: DEBUG nova.virt.vmwareapi.images [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Downloaded image file data 0422123c-16ce-4c9d-b1c6-0281fe0e54ad to vmware_temp/7eeef8c1-5ccb-4b80-9ac3-c1526fd140d5/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/tmp-sparse.vmdk on the data store datastore2 {{(pid=62499) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 547.886964] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Caching image {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 547.886964] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Copying Virtual Disk [datastore2] vmware_temp/7eeef8c1-5ccb-4b80-9ac3-c1526fd140d5/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/tmp-sparse.vmdk to [datastore2] vmware_temp/7eeef8c1-5ccb-4b80-9ac3-c1526fd140d5/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 547.887467] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f2758c0-875a-48a9-8e44-44fbff818c0f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 547.892156] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-c99e5aad-47dd-43da-899d-cb7a816f1d27 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 547.924995] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a9c77bc-130d-40a7-afde-8f9457d3a883 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 547.931470] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 547.931470] env[62499]: value = "task-1335641" [ 547.931470] env[62499]: _type = "Task" [ 547.931470] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 547.938654] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f572cfb9-ec28-42d7-90ef-3293e96dc4c4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 547.945825] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335641, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 547.957523] env[62499]: DEBUG nova.compute.provider_tree [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 547.983281] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Acquiring lock "27103ce2-0962-4fce-9331-d74179b0510c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 547.983631] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Lock "27103ce2-0962-4fce-9331-d74179b0510c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 548.119464] env[62499]: INFO nova.compute.manager [-] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Took 1.11 seconds to deallocate network for instance. [ 548.122718] env[62499]: DEBUG nova.compute.claims [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 548.122928] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 548.384205] env[62499]: DEBUG oslo_concurrency.lockutils [req-c92936c2-0762-4aeb-8c9d-ae76e693de8c req-cfde7e9e-4ddf-4ec7-b9b6-8f954b0bc358 service nova] Releasing lock "refresh_cache-904c5d79-86f3-4d5b-b8ba-7e3d515b32c0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 548.451028] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335641, 'name': CopyVirtualDisk_Task} progress is 100%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 548.460333] env[62499]: DEBUG nova.scheduler.client.report [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 548.486482] env[62499]: DEBUG nova.compute.manager [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 548.717761] env[62499]: DEBUG nova.compute.manager [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 548.763247] env[62499]: DEBUG nova.virt.hardware [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 548.763585] env[62499]: DEBUG nova.virt.hardware [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 548.763585] env[62499]: DEBUG nova.virt.hardware [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 548.763752] env[62499]: DEBUG nova.virt.hardware [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 548.766711] env[62499]: DEBUG nova.virt.hardware [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 548.766959] env[62499]: DEBUG nova.virt.hardware [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 548.767041] env[62499]: DEBUG nova.virt.hardware [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 548.767259] env[62499]: DEBUG nova.virt.hardware [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 548.767439] env[62499]: DEBUG nova.virt.hardware [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 548.767605] env[62499]: DEBUG nova.virt.hardware [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 548.768023] env[62499]: DEBUG nova.virt.hardware [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 548.768737] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f06da999-753d-4f0e-9487-0968a0457306 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.781821] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-41efb47e-2d9d-49bb-9b3f-dd467ad69524 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.909820] env[62499]: DEBUG nova.network.neutron [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Successfully created port: eff92d51-a578-421d-98e9-d5ffbfb11bb1 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 548.944823] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335641, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.73193} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 548.945327] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Copied Virtual Disk [datastore2] vmware_temp/7eeef8c1-5ccb-4b80-9ac3-c1526fd140d5/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/tmp-sparse.vmdk to [datastore2] vmware_temp/7eeef8c1-5ccb-4b80-9ac3-c1526fd140d5/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 548.945895] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Deleting the datastore file [datastore2] vmware_temp/7eeef8c1-5ccb-4b80-9ac3-c1526fd140d5/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/tmp-sparse.vmdk {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 548.946293] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-f1afe4f0-3472-4202-958f-d7422d634d17 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.955716] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 548.955716] env[62499]: value = "task-1335642" [ 548.955716] env[62499]: _type = "Task" [ 548.955716] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 548.967019] env[62499]: DEBUG oslo_concurrency.lockutils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.286s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 548.967572] env[62499]: DEBUG nova.compute.manager [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 548.970054] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335642, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 548.970930] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 6.176s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 549.021268] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 549.470073] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335642, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.024422} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 549.473332] env[62499]: DEBUG nova.compute.utils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 549.473332] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Deleted the datastore file {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 549.473521] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Moving file from [datastore2] vmware_temp/7eeef8c1-5ccb-4b80-9ac3-c1526fd140d5/0422123c-16ce-4c9d-b1c6-0281fe0e54ad to [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad. {{(pid=62499) file_move /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:334}} [ 549.474846] env[62499]: DEBUG nova.compute.manager [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 549.474846] env[62499]: DEBUG nova.network.neutron [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 549.477370] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.MoveDatastoreFile_Task with opID=oslo.vmware-eb3b8158-2a90-4cba-bc6e-24c7faafda0c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.493693] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 549.493693] env[62499]: value = "task-1335643" [ 549.493693] env[62499]: _type = "Task" [ 549.493693] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 549.503651] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335643, 'name': MoveDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 549.667512] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ad777ef-647a-41c9-8b4b-0314f2327645 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.677558] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0cb60c1b-1edc-4c61-82e9-94b9c915caaf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.710435] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7be35356-e999-4c08-a247-25428533fcf3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.718461] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f0aaaf19-c046-494e-b6ea-45c827d42108 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.724786] env[62499]: DEBUG nova.policy [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'bc4be38552f7413e9ed97d5bd407fdcd', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c3bcaeb2afd44f9a8621fbb43f65a1f7', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 549.737155] env[62499]: DEBUG nova.compute.provider_tree [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 549.975072] env[62499]: DEBUG nova.compute.manager [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 550.004280] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335643, 'name': MoveDatastoreFile_Task, 'duration_secs': 0.026455} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 550.004280] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] File moved {{(pid=62499) file_move /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:346}} [ 550.004280] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Cleaning up location [datastore2] vmware_temp/7eeef8c1-5ccb-4b80-9ac3-c1526fd140d5 {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:640}} [ 550.004280] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Deleting the datastore file [datastore2] vmware_temp/7eeef8c1-5ccb-4b80-9ac3-c1526fd140d5 {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 550.004280] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-52a856a6-4adf-491b-8940-4d14614ab2b6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 550.011335] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 550.011335] env[62499]: value = "task-1335644" [ 550.011335] env[62499]: _type = "Task" [ 550.011335] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 550.022067] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335644, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 550.256157] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquiring lock "2b7122b2-136d-4e0c-97eb-155b33905436" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 550.256157] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "2b7122b2-136d-4e0c-97eb-155b33905436" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 550.283169] env[62499]: ERROR nova.scheduler.client.report [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [req-7b911a02-e380-4d24-a8cb-499a580fa818] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID 38a6c941-9755-44af-8da8-6ef6281d0157. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-7b911a02-e380-4d24-a8cb-499a580fa818"}]}: nova.exception.PortBindingFailed: Binding failed for port d19aae75-6444-4c54-9764-32b1260ecf19, please check neutron logs for more information. [ 550.303613] env[62499]: DEBUG nova.scheduler.client.report [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Refreshing inventories for resource provider 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:818}} [ 550.325586] env[62499]: DEBUG nova.scheduler.client.report [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Updating ProviderTree inventory for provider 38a6c941-9755-44af-8da8-6ef6281d0157 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:782}} [ 550.325925] env[62499]: DEBUG nova.compute.provider_tree [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 550.339727] env[62499]: DEBUG nova.scheduler.client.report [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Refreshing aggregate associations for resource provider 38a6c941-9755-44af-8da8-6ef6281d0157, aggregates: None {{(pid=62499) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:827}} [ 550.359128] env[62499]: DEBUG nova.scheduler.client.report [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Refreshing trait associations for resource provider 38a6c941-9755-44af-8da8-6ef6281d0157, traits: COMPUTE_NET_ATTACH_INTERFACE,HW_ARCH_X86_64,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NODE {{(pid=62499) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:839}} [ 550.523477] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335644, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.033058} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 550.523771] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Deleted the datastore file {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 550.524799] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-4c372bd6-4500-450a-afc5-3aa3460b9228 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 550.532463] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21eefdfb-397a-4d1e-9e69-5668e07350b7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 550.538283] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 550.538283] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52719a61-a10f-b471-4aa9-d1fb7583ca80" [ 550.538283] env[62499]: _type = "Task" [ 550.538283] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 550.546218] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-10cbe957-5259-4d18-8b31-e6c0c318e243 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 550.556673] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52719a61-a10f-b471-4aa9-d1fb7583ca80, 'name': SearchDatastore_Task, 'duration_secs': 0.009166} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 550.557335] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Releasing lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 550.557608] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore2] ddd212aa-9f4a-4b9d-aa60-cd4b64580f84/ddd212aa-9f4a-4b9d-aa60-cd4b64580f84.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 550.558031] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-9d3db24d-6f45-4410-89ef-3a2aa3fdfdbf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 550.592342] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12422a97-bc9e-4826-80b8-3006d91e0bef {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 550.596226] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 550.596226] env[62499]: value = "task-1335645" [ 550.596226] env[62499]: _type = "Task" [ 550.596226] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 550.602798] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d6db78e7-5034-4657-bf43-b2b4cd3e041a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 550.610316] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335645, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 550.624285] env[62499]: DEBUG nova.compute.provider_tree [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 550.759228] env[62499]: DEBUG nova.compute.manager [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 550.927056] env[62499]: DEBUG nova.network.neutron [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Successfully created port: f71924e4-d6df-4e41-b251-783bd1c767a7 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 550.989498] env[62499]: DEBUG nova.compute.manager [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 551.034444] env[62499]: DEBUG nova.virt.hardware [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 551.034673] env[62499]: DEBUG nova.virt.hardware [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 551.034825] env[62499]: DEBUG nova.virt.hardware [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 551.035235] env[62499]: DEBUG nova.virt.hardware [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 551.037331] env[62499]: DEBUG nova.virt.hardware [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 551.037331] env[62499]: DEBUG nova.virt.hardware [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 551.037331] env[62499]: DEBUG nova.virt.hardware [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 551.037331] env[62499]: DEBUG nova.virt.hardware [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 551.040043] env[62499]: DEBUG nova.virt.hardware [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 551.040043] env[62499]: DEBUG nova.virt.hardware [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 551.040043] env[62499]: DEBUG nova.virt.hardware [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 551.040043] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0bfc3487-8fc4-4bf8-86e3-733dc153df25 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 551.055635] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6ddf641-6417-4f28-8423-1afc7f951a3a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 551.106272] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335645, 'name': CopyVirtualDisk_Task} progress is 100%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 551.160436] env[62499]: ERROR nova.scheduler.client.report [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [req-4f3f0d65-7746-481a-9e98-8c8de563e750] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID 38a6c941-9755-44af-8da8-6ef6281d0157. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-4f3f0d65-7746-481a-9e98-8c8de563e750"}]}: nova.exception.PortBindingFailed: Binding failed for port d19aae75-6444-4c54-9764-32b1260ecf19, please check neutron logs for more information. [ 551.185597] env[62499]: DEBUG nova.scheduler.client.report [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Refreshing inventories for resource provider 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:818}} [ 551.214024] env[62499]: DEBUG nova.scheduler.client.report [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Updating ProviderTree inventory for provider 38a6c941-9755-44af-8da8-6ef6281d0157 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:782}} [ 551.214024] env[62499]: DEBUG nova.compute.provider_tree [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 551.237308] env[62499]: DEBUG nova.scheduler.client.report [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Refreshing aggregate associations for resource provider 38a6c941-9755-44af-8da8-6ef6281d0157, aggregates: None {{(pid=62499) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:827}} [ 551.260157] env[62499]: DEBUG nova.scheduler.client.report [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Refreshing trait associations for resource provider 38a6c941-9755-44af-8da8-6ef6281d0157, traits: COMPUTE_NET_ATTACH_INTERFACE,HW_ARCH_X86_64,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NODE {{(pid=62499) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:839}} [ 551.294397] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 551.331807] env[62499]: DEBUG nova.compute.manager [req-3c522e0c-5c61-460b-b6ba-1436c483e787 req-6ff76941-17f0-428d-9055-f35ff09869f3 service nova] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Received event network-vif-deleted-d253c0f7-c107-4caf-8d19-68e503db563f {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 551.468014] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c273bcf-ea13-446a-9aec-66c128d6c31c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 551.476172] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7af01be3-0383-495b-81c7-298db2663db7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 551.509270] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-133f3f86-49b9-4337-a942-4a997d3abc81 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 551.518355] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-438e5245-4dc0-402b-9ac1-ea233180134b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 551.535070] env[62499]: DEBUG nova.compute.provider_tree [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 551.545949] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Acquiring lock "76cf7f28-7413-4b06-bd30-ae22bbc2999e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 551.546206] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Lock "76cf7f28-7413-4b06-bd30-ae22bbc2999e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 551.616792] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335645, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.511229} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 551.616792] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore2] ddd212aa-9f4a-4b9d-aa60-cd4b64580f84/ddd212aa-9f4a-4b9d-aa60-cd4b64580f84.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 551.616792] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Extending root virtual disk to 1048576 {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 551.616792] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-b615387f-da4f-4e00-8d8b-79743e3798ee {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 551.623754] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 551.623754] env[62499]: value = "task-1335646" [ 551.623754] env[62499]: _type = "Task" [ 551.623754] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 551.639420] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335646, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 551.975577] env[62499]: ERROR nova.compute.manager [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 900edb48-3a43-497c-9834-e3a23a708ba3, please check neutron logs for more information. [ 551.975577] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 551.975577] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 551.975577] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 551.975577] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 551.975577] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 551.975577] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 551.975577] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 551.975577] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 551.975577] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 551.975577] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 551.975577] env[62499]: ERROR nova.compute.manager raise self.value [ 551.975577] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 551.975577] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 551.975577] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 551.975577] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 551.976101] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 551.976101] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 551.976101] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 900edb48-3a43-497c-9834-e3a23a708ba3, please check neutron logs for more information. [ 551.976101] env[62499]: ERROR nova.compute.manager [ 551.976101] env[62499]: Traceback (most recent call last): [ 551.976101] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 551.976101] env[62499]: listener.cb(fileno) [ 551.976101] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 551.976101] env[62499]: result = function(*args, **kwargs) [ 551.976101] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 551.976101] env[62499]: return func(*args, **kwargs) [ 551.976101] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 551.976101] env[62499]: raise e [ 551.976101] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 551.976101] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 551.976101] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 551.976101] env[62499]: created_port_ids = self._update_ports_for_instance( [ 551.976101] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 551.976101] env[62499]: with excutils.save_and_reraise_exception(): [ 551.976101] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 551.976101] env[62499]: self.force_reraise() [ 551.976101] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 551.976101] env[62499]: raise self.value [ 551.976101] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 551.976101] env[62499]: updated_port = self._update_port( [ 551.976101] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 551.976101] env[62499]: _ensure_no_port_binding_failure(port) [ 551.976101] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 551.976101] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 551.976823] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 900edb48-3a43-497c-9834-e3a23a708ba3, please check neutron logs for more information. [ 551.976823] env[62499]: Removing descriptor: 15 [ 551.976823] env[62499]: ERROR nova.compute.manager [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 900edb48-3a43-497c-9834-e3a23a708ba3, please check neutron logs for more information. [ 551.976823] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Traceback (most recent call last): [ 551.976823] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 551.976823] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] yield resources [ 551.976823] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 551.976823] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] self.driver.spawn(context, instance, image_meta, [ 551.976823] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 551.976823] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] self._vmops.spawn(context, instance, image_meta, injected_files, [ 551.976823] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 551.976823] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] vm_ref = self.build_virtual_machine(instance, [ 551.977297] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 551.977297] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] vif_infos = vmwarevif.get_vif_info(self._session, [ 551.977297] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 551.977297] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] for vif in network_info: [ 551.977297] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 551.977297] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] return self._sync_wrapper(fn, *args, **kwargs) [ 551.977297] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 551.977297] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] self.wait() [ 551.977297] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 551.977297] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] self[:] = self._gt.wait() [ 551.977297] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 551.977297] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] return self._exit_event.wait() [ 551.977297] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 551.977622] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] result = hub.switch() [ 551.977622] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 551.977622] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] return self.greenlet.switch() [ 551.977622] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 551.977622] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] result = function(*args, **kwargs) [ 551.977622] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 551.977622] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] return func(*args, **kwargs) [ 551.977622] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 551.977622] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] raise e [ 551.977622] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 551.977622] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] nwinfo = self.network_api.allocate_for_instance( [ 551.977622] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 551.977622] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] created_port_ids = self._update_ports_for_instance( [ 551.977977] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 551.977977] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] with excutils.save_and_reraise_exception(): [ 551.977977] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 551.977977] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] self.force_reraise() [ 551.977977] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 551.977977] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] raise self.value [ 551.977977] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 551.977977] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] updated_port = self._update_port( [ 551.977977] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 551.977977] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] _ensure_no_port_binding_failure(port) [ 551.977977] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 551.977977] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] raise exception.PortBindingFailed(port_id=port['id']) [ 551.978321] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] nova.exception.PortBindingFailed: Binding failed for port 900edb48-3a43-497c-9834-e3a23a708ba3, please check neutron logs for more information. [ 551.978321] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] [ 551.978321] env[62499]: INFO nova.compute.manager [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Terminating instance [ 551.981672] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Acquiring lock "refresh_cache-812c740f-eae7-4f6d-9e0d-8c16a4685a50" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 551.981869] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Acquired lock "refresh_cache-812c740f-eae7-4f6d-9e0d-8c16a4685a50" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 551.982072] env[62499]: DEBUG nova.network.neutron [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 552.055910] env[62499]: DEBUG nova.compute.manager [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 552.079792] env[62499]: DEBUG nova.scheduler.client.report [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Updated inventory for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with generation 14 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:971}} [ 552.079792] env[62499]: DEBUG nova.compute.provider_tree [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Updating resource provider 38a6c941-9755-44af-8da8-6ef6281d0157 generation from 14 to 15 during operation: update_inventory {{(pid=62499) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 552.079792] env[62499]: DEBUG nova.compute.provider_tree [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 552.139034] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335646, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.072745} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 552.139205] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Extended root virtual disk {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 552.140015] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75f6239b-4f83-466c-a84c-af55c4b7cae8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.165847] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Reconfiguring VM instance instance-00000004 to attach disk [datastore2] ddd212aa-9f4a-4b9d-aa60-cd4b64580f84/ddd212aa-9f4a-4b9d-aa60-cd4b64580f84.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 552.166597] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-c9b4623b-dc26-4667-9f79-41f2a824a06f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.189639] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 552.189639] env[62499]: value = "task-1335647" [ 552.189639] env[62499]: _type = "Task" [ 552.189639] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 552.194500] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335647, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 552.561670] env[62499]: DEBUG nova.network.neutron [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 552.581476] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 552.587503] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 3.617s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 552.590326] env[62499]: ERROR nova.compute.manager [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d19aae75-6444-4c54-9764-32b1260ecf19, please check neutron logs for more information. [ 552.590326] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Traceback (most recent call last): [ 552.590326] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 552.590326] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] self.driver.spawn(context, instance, image_meta, [ 552.590326] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 552.590326] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 552.590326] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 552.590326] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] vm_ref = self.build_virtual_machine(instance, [ 552.590326] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 552.590326] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] vif_infos = vmwarevif.get_vif_info(self._session, [ 552.590326] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 552.590691] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] for vif in network_info: [ 552.590691] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 552.590691] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] return self._sync_wrapper(fn, *args, **kwargs) [ 552.590691] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 552.590691] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] self.wait() [ 552.590691] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 552.590691] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] self[:] = self._gt.wait() [ 552.590691] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 552.590691] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] return self._exit_event.wait() [ 552.590691] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 552.590691] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] result = hub.switch() [ 552.590691] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 552.590691] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] return self.greenlet.switch() [ 552.591079] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 552.591079] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] result = function(*args, **kwargs) [ 552.591079] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 552.591079] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] return func(*args, **kwargs) [ 552.591079] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 552.591079] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] raise e [ 552.591079] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 552.591079] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] nwinfo = self.network_api.allocate_for_instance( [ 552.591079] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 552.591079] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] created_port_ids = self._update_ports_for_instance( [ 552.591079] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 552.591079] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] with excutils.save_and_reraise_exception(): [ 552.591079] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 552.591453] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] self.force_reraise() [ 552.591453] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 552.591453] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] raise self.value [ 552.591453] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 552.591453] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] updated_port = self._update_port( [ 552.591453] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 552.591453] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] _ensure_no_port_binding_failure(port) [ 552.591453] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 552.591453] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] raise exception.PortBindingFailed(port_id=port['id']) [ 552.591453] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] nova.exception.PortBindingFailed: Binding failed for port d19aae75-6444-4c54-9764-32b1260ecf19, please check neutron logs for more information. [ 552.591453] env[62499]: ERROR nova.compute.manager [instance: ebd5745d-f073-429c-b609-a7520a539f2b] [ 552.591770] env[62499]: DEBUG nova.compute.utils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Binding failed for port d19aae75-6444-4c54-9764-32b1260ecf19, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 552.591770] env[62499]: DEBUG oslo_concurrency.lockutils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 8.258s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 552.602064] env[62499]: DEBUG nova.compute.manager [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Build of instance ebd5745d-f073-429c-b609-a7520a539f2b was re-scheduled: Binding failed for port d19aae75-6444-4c54-9764-32b1260ecf19, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 552.602064] env[62499]: DEBUG nova.compute.manager [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 552.602064] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Acquiring lock "refresh_cache-ebd5745d-f073-429c-b609-a7520a539f2b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 552.602064] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Acquired lock "refresh_cache-ebd5745d-f073-429c-b609-a7520a539f2b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 552.602298] env[62499]: DEBUG nova.network.neutron [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 552.701062] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335647, 'name': ReconfigVM_Task, 'duration_secs': 0.282052} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 552.702294] env[62499]: DEBUG nova.network.neutron [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 552.706304] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Reconfigured VM instance instance-00000004 to attach disk [datastore2] ddd212aa-9f4a-4b9d-aa60-cd4b64580f84/ddd212aa-9f4a-4b9d-aa60-cd4b64580f84.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 552.706304] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-15d973bd-75f1-41e6-a3bb-cbc14a383c17 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.718166] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 552.718166] env[62499]: value = "task-1335648" [ 552.718166] env[62499]: _type = "Task" [ 552.718166] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 552.732303] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335648, 'name': Rename_Task} progress is 6%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 552.836689] env[62499]: ERROR nova.compute.manager [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port eff92d51-a578-421d-98e9-d5ffbfb11bb1, please check neutron logs for more information. [ 552.836689] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 552.836689] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 552.836689] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 552.836689] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 552.836689] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 552.836689] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 552.836689] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 552.836689] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 552.836689] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 552.836689] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 552.836689] env[62499]: ERROR nova.compute.manager raise self.value [ 552.836689] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 552.836689] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 552.836689] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 552.836689] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 552.837804] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 552.837804] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 552.837804] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port eff92d51-a578-421d-98e9-d5ffbfb11bb1, please check neutron logs for more information. [ 552.837804] env[62499]: ERROR nova.compute.manager [ 552.837804] env[62499]: Traceback (most recent call last): [ 552.837804] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 552.837804] env[62499]: listener.cb(fileno) [ 552.837804] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 552.837804] env[62499]: result = function(*args, **kwargs) [ 552.837804] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 552.837804] env[62499]: return func(*args, **kwargs) [ 552.837804] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 552.837804] env[62499]: raise e [ 552.837804] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 552.837804] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 552.837804] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 552.837804] env[62499]: created_port_ids = self._update_ports_for_instance( [ 552.837804] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 552.837804] env[62499]: with excutils.save_and_reraise_exception(): [ 552.837804] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 552.837804] env[62499]: self.force_reraise() [ 552.837804] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 552.837804] env[62499]: raise self.value [ 552.837804] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 552.837804] env[62499]: updated_port = self._update_port( [ 552.837804] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 552.837804] env[62499]: _ensure_no_port_binding_failure(port) [ 552.837804] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 552.837804] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 552.838533] env[62499]: nova.exception.PortBindingFailed: Binding failed for port eff92d51-a578-421d-98e9-d5ffbfb11bb1, please check neutron logs for more information. [ 552.838533] env[62499]: Removing descriptor: 17 [ 552.838533] env[62499]: ERROR nova.compute.manager [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port eff92d51-a578-421d-98e9-d5ffbfb11bb1, please check neutron logs for more information. [ 552.838533] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Traceback (most recent call last): [ 552.838533] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 552.838533] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] yield resources [ 552.838533] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 552.838533] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] self.driver.spawn(context, instance, image_meta, [ 552.838533] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 552.838533] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 552.838533] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 552.838533] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] vm_ref = self.build_virtual_machine(instance, [ 552.838838] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 552.838838] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] vif_infos = vmwarevif.get_vif_info(self._session, [ 552.838838] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 552.838838] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] for vif in network_info: [ 552.838838] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 552.838838] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] return self._sync_wrapper(fn, *args, **kwargs) [ 552.838838] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 552.838838] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] self.wait() [ 552.838838] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 552.838838] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] self[:] = self._gt.wait() [ 552.838838] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 552.838838] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] return self._exit_event.wait() [ 552.838838] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 552.839180] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] result = hub.switch() [ 552.839180] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 552.839180] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] return self.greenlet.switch() [ 552.839180] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 552.839180] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] result = function(*args, **kwargs) [ 552.839180] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 552.839180] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] return func(*args, **kwargs) [ 552.839180] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 552.839180] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] raise e [ 552.839180] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 552.839180] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] nwinfo = self.network_api.allocate_for_instance( [ 552.839180] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 552.839180] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] created_port_ids = self._update_ports_for_instance( [ 552.839505] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 552.839505] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] with excutils.save_and_reraise_exception(): [ 552.839505] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 552.839505] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] self.force_reraise() [ 552.839505] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 552.839505] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] raise self.value [ 552.839505] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 552.839505] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] updated_port = self._update_port( [ 552.839505] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 552.839505] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] _ensure_no_port_binding_failure(port) [ 552.839505] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 552.839505] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] raise exception.PortBindingFailed(port_id=port['id']) [ 552.839870] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] nova.exception.PortBindingFailed: Binding failed for port eff92d51-a578-421d-98e9-d5ffbfb11bb1, please check neutron logs for more information. [ 552.839870] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] [ 552.839870] env[62499]: INFO nova.compute.manager [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Terminating instance [ 552.844152] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Acquiring lock "refresh_cache-48fc0cbf-18a6-4bbc-8455-93800b911d6a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 552.844152] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Acquired lock "refresh_cache-48fc0cbf-18a6-4bbc-8455-93800b911d6a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 552.844152] env[62499]: DEBUG nova.network.neutron [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 552.890964] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Acquiring lock "8d45497d-ba77-45b1-91f3-719b2689bee3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 552.891256] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Lock "8d45497d-ba77-45b1-91f3-719b2689bee3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.004s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 553.119518] env[62499]: DEBUG nova.compute.manager [req-429a4e0f-f655-4d7e-b666-6dde4999b12d req-4cd934e4-e232-4060-b2d7-ba13f3d7cd50 service nova] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Received event network-changed-900edb48-3a43-497c-9834-e3a23a708ba3 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 553.119724] env[62499]: DEBUG nova.compute.manager [req-429a4e0f-f655-4d7e-b666-6dde4999b12d req-4cd934e4-e232-4060-b2d7-ba13f3d7cd50 service nova] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Refreshing instance network info cache due to event network-changed-900edb48-3a43-497c-9834-e3a23a708ba3. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 553.119857] env[62499]: DEBUG oslo_concurrency.lockutils [req-429a4e0f-f655-4d7e-b666-6dde4999b12d req-4cd934e4-e232-4060-b2d7-ba13f3d7cd50 service nova] Acquiring lock "refresh_cache-812c740f-eae7-4f6d-9e0d-8c16a4685a50" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 553.167337] env[62499]: DEBUG nova.network.neutron [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 553.216955] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Releasing lock "refresh_cache-812c740f-eae7-4f6d-9e0d-8c16a4685a50" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 553.216955] env[62499]: DEBUG nova.compute.manager [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 553.217156] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 553.220228] env[62499]: DEBUG oslo_concurrency.lockutils [req-429a4e0f-f655-4d7e-b666-6dde4999b12d req-4cd934e4-e232-4060-b2d7-ba13f3d7cd50 service nova] Acquired lock "refresh_cache-812c740f-eae7-4f6d-9e0d-8c16a4685a50" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 553.220559] env[62499]: DEBUG nova.network.neutron [req-429a4e0f-f655-4d7e-b666-6dde4999b12d req-4cd934e4-e232-4060-b2d7-ba13f3d7cd50 service nova] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Refreshing network info cache for port 900edb48-3a43-497c-9834-e3a23a708ba3 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 553.221706] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-24c41446-155c-4c1d-831b-12f6f70198f0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 553.233593] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335648, 'name': Rename_Task, 'duration_secs': 0.142749} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 553.235168] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Powering on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 553.235452] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-06a83243-ab01-462d-9523-f86844bb74d9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 553.240531] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fae7fd5d-54fd-4b4a-8399-d702800c7826 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 553.266408] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 553.266408] env[62499]: value = "task-1335649" [ 553.266408] env[62499]: _type = "Task" [ 553.266408] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 553.276032] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 812c740f-eae7-4f6d-9e0d-8c16a4685a50 could not be found. [ 553.276032] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 553.276032] env[62499]: INFO nova.compute.manager [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Took 0.06 seconds to destroy the instance on the hypervisor. [ 553.276259] env[62499]: DEBUG oslo.service.loopingcall [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 553.279562] env[62499]: DEBUG nova.compute.manager [-] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 553.279664] env[62499]: DEBUG nova.network.neutron [-] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 553.284706] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335649, 'name': PowerOnVM_Task} progress is 33%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 553.310469] env[62499]: DEBUG nova.network.neutron [-] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 553.367059] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5e97d61-ceee-4760-adeb-648116941a61 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 553.376959] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96eeba65-70a7-402c-9b68-e5fb49cb6024 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 553.381164] env[62499]: DEBUG nova.network.neutron [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 553.412955] env[62499]: DEBUG nova.network.neutron [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 553.415368] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c672747-f239-45e8-a6f8-ecd719f0c281 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 553.424244] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c67b6a5-1f9a-403e-96b4-e2ac555da59a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 553.439064] env[62499]: DEBUG nova.compute.provider_tree [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 553.505645] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Acquiring lock "7c5b9fab-bb90-4c70-b445-fe0b637dbd4a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 553.506048] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Lock "7c5b9fab-bb90-4c70-b445-fe0b637dbd4a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 553.578601] env[62499]: DEBUG nova.network.neutron [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 553.622768] env[62499]: DEBUG oslo_concurrency.lockutils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Acquiring lock "3706243f-4768-4b44-9a5b-730bf2b18974" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 553.623718] env[62499]: DEBUG oslo_concurrency.lockutils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Lock "3706243f-4768-4b44-9a5b-730bf2b18974" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 553.763972] env[62499]: DEBUG nova.network.neutron [req-429a4e0f-f655-4d7e-b666-6dde4999b12d req-4cd934e4-e232-4060-b2d7-ba13f3d7cd50 service nova] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 553.781026] env[62499]: DEBUG oslo_vmware.api [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335649, 'name': PowerOnVM_Task, 'duration_secs': 0.435104} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 553.781026] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Powered on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 553.781026] env[62499]: INFO nova.compute.manager [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Took 9.68 seconds to spawn the instance on the hypervisor. [ 553.781026] env[62499]: DEBUG nova.compute.manager [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 553.781245] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ad45acc-2cb4-4cfe-a258-1b493ea4220d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 553.814440] env[62499]: DEBUG nova.network.neutron [-] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 553.884763] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Releasing lock "refresh_cache-ebd5745d-f073-429c-b609-a7520a539f2b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 553.885102] env[62499]: DEBUG nova.compute.manager [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 553.885854] env[62499]: DEBUG nova.compute.manager [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 553.885854] env[62499]: DEBUG nova.network.neutron [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 553.932281] env[62499]: DEBUG nova.network.neutron [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 553.943933] env[62499]: DEBUG nova.scheduler.client.report [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 553.960882] env[62499]: DEBUG nova.network.neutron [req-429a4e0f-f655-4d7e-b666-6dde4999b12d req-4cd934e4-e232-4060-b2d7-ba13f3d7cd50 service nova] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 554.085518] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Releasing lock "refresh_cache-48fc0cbf-18a6-4bbc-8455-93800b911d6a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 554.085518] env[62499]: DEBUG nova.compute.manager [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 554.085518] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 554.085518] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8d1bb8c9-4323-4541-965c-53e7ded0e3f4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 554.097956] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d406cb3-0bf3-4ecf-8e61-b89a85e81036 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 554.127610] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 48fc0cbf-18a6-4bbc-8455-93800b911d6a could not be found. [ 554.127610] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 554.127610] env[62499]: INFO nova.compute.manager [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Took 0.04 seconds to destroy the instance on the hypervisor. [ 554.127709] env[62499]: DEBUG oslo.service.loopingcall [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 554.129642] env[62499]: ERROR nova.compute.manager [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f71924e4-d6df-4e41-b251-783bd1c767a7, please check neutron logs for more information. [ 554.129642] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 554.129642] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 554.129642] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 554.129642] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 554.129642] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 554.129642] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 554.129642] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 554.129642] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 554.129642] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 554.129642] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 554.129642] env[62499]: ERROR nova.compute.manager raise self.value [ 554.129642] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 554.129642] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 554.129642] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 554.129642] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 554.130077] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 554.130077] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 554.130077] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f71924e4-d6df-4e41-b251-783bd1c767a7, please check neutron logs for more information. [ 554.130077] env[62499]: ERROR nova.compute.manager [ 554.130077] env[62499]: Traceback (most recent call last): [ 554.130077] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 554.130077] env[62499]: listener.cb(fileno) [ 554.130077] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 554.130077] env[62499]: result = function(*args, **kwargs) [ 554.130077] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 554.130077] env[62499]: return func(*args, **kwargs) [ 554.130077] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 554.130077] env[62499]: raise e [ 554.130077] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 554.130077] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 554.130077] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 554.130077] env[62499]: created_port_ids = self._update_ports_for_instance( [ 554.130077] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 554.130077] env[62499]: with excutils.save_and_reraise_exception(): [ 554.130077] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 554.130077] env[62499]: self.force_reraise() [ 554.130077] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 554.130077] env[62499]: raise self.value [ 554.130077] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 554.130077] env[62499]: updated_port = self._update_port( [ 554.130077] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 554.130077] env[62499]: _ensure_no_port_binding_failure(port) [ 554.130077] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 554.130077] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 554.131732] env[62499]: nova.exception.PortBindingFailed: Binding failed for port f71924e4-d6df-4e41-b251-783bd1c767a7, please check neutron logs for more information. [ 554.131732] env[62499]: Removing descriptor: 16 [ 554.131732] env[62499]: DEBUG nova.compute.manager [-] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 554.131732] env[62499]: DEBUG nova.network.neutron [-] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 554.131732] env[62499]: ERROR nova.compute.manager [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f71924e4-d6df-4e41-b251-783bd1c767a7, please check neutron logs for more information. [ 554.131732] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Traceback (most recent call last): [ 554.131732] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 554.131732] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] yield resources [ 554.131732] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 554.131732] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] self.driver.spawn(context, instance, image_meta, [ 554.131732] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 554.133023] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 554.133023] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 554.133023] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] vm_ref = self.build_virtual_machine(instance, [ 554.133023] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 554.133023] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] vif_infos = vmwarevif.get_vif_info(self._session, [ 554.133023] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 554.133023] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] for vif in network_info: [ 554.133023] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 554.133023] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] return self._sync_wrapper(fn, *args, **kwargs) [ 554.133023] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 554.133023] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] self.wait() [ 554.133023] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 554.133023] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] self[:] = self._gt.wait() [ 554.134244] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 554.134244] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] return self._exit_event.wait() [ 554.134244] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 554.134244] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] result = hub.switch() [ 554.134244] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 554.134244] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] return self.greenlet.switch() [ 554.134244] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 554.134244] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] result = function(*args, **kwargs) [ 554.134244] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 554.134244] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] return func(*args, **kwargs) [ 554.134244] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 554.134244] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] raise e [ 554.134244] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 554.135000] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] nwinfo = self.network_api.allocate_for_instance( [ 554.135000] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 554.135000] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] created_port_ids = self._update_ports_for_instance( [ 554.135000] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 554.135000] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] with excutils.save_and_reraise_exception(): [ 554.135000] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 554.135000] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] self.force_reraise() [ 554.135000] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 554.135000] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] raise self.value [ 554.135000] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 554.135000] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] updated_port = self._update_port( [ 554.135000] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 554.135000] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] _ensure_no_port_binding_failure(port) [ 554.136282] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 554.136282] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] raise exception.PortBindingFailed(port_id=port['id']) [ 554.136282] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] nova.exception.PortBindingFailed: Binding failed for port f71924e4-d6df-4e41-b251-783bd1c767a7, please check neutron logs for more information. [ 554.136282] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] [ 554.136282] env[62499]: INFO nova.compute.manager [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Terminating instance [ 554.136282] env[62499]: DEBUG oslo_concurrency.lockutils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Acquiring lock "refresh_cache-99f77fa6-a625-4771-83c9-012f75ed1c7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 554.136282] env[62499]: DEBUG oslo_concurrency.lockutils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Acquired lock "refresh_cache-99f77fa6-a625-4771-83c9-012f75ed1c7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 554.136602] env[62499]: DEBUG nova.network.neutron [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 554.148042] env[62499]: DEBUG nova.network.neutron [-] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 554.307571] env[62499]: INFO nova.compute.manager [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Took 18.04 seconds to build instance. [ 554.320270] env[62499]: INFO nova.compute.manager [-] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Took 1.04 seconds to deallocate network for instance. [ 554.323788] env[62499]: DEBUG nova.compute.claims [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 554.323788] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 554.438890] env[62499]: DEBUG nova.network.neutron [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 554.451433] env[62499]: DEBUG oslo_concurrency.lockutils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.861s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.452832] env[62499]: ERROR nova.compute.manager [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 8fd4fe87-772a-4544-b201-6fed68efdb0a, please check neutron logs for more information. [ 554.452832] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Traceback (most recent call last): [ 554.452832] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 554.452832] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] self.driver.spawn(context, instance, image_meta, [ 554.452832] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 554.452832] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] self._vmops.spawn(context, instance, image_meta, injected_files, [ 554.452832] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 554.452832] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] vm_ref = self.build_virtual_machine(instance, [ 554.452832] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 554.452832] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] vif_infos = vmwarevif.get_vif_info(self._session, [ 554.452832] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 554.453239] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] for vif in network_info: [ 554.453239] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 554.453239] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] return self._sync_wrapper(fn, *args, **kwargs) [ 554.453239] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 554.453239] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] self.wait() [ 554.453239] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 554.453239] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] self[:] = self._gt.wait() [ 554.453239] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 554.453239] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] return self._exit_event.wait() [ 554.453239] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 554.453239] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] result = hub.switch() [ 554.453239] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 554.453239] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] return self.greenlet.switch() [ 554.453590] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 554.453590] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] result = function(*args, **kwargs) [ 554.453590] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 554.453590] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] return func(*args, **kwargs) [ 554.453590] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 554.453590] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] raise e [ 554.453590] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 554.453590] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] nwinfo = self.network_api.allocate_for_instance( [ 554.453590] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 554.453590] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] created_port_ids = self._update_ports_for_instance( [ 554.453590] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 554.453590] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] with excutils.save_and_reraise_exception(): [ 554.453590] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 554.453951] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] self.force_reraise() [ 554.453951] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 554.453951] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] raise self.value [ 554.453951] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 554.453951] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] updated_port = self._update_port( [ 554.453951] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 554.453951] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] _ensure_no_port_binding_failure(port) [ 554.453951] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 554.453951] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] raise exception.PortBindingFailed(port_id=port['id']) [ 554.453951] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] nova.exception.PortBindingFailed: Binding failed for port 8fd4fe87-772a-4544-b201-6fed68efdb0a, please check neutron logs for more information. [ 554.453951] env[62499]: ERROR nova.compute.manager [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] [ 554.454297] env[62499]: DEBUG nova.compute.utils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Binding failed for port 8fd4fe87-772a-4544-b201-6fed68efdb0a, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 554.457956] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 9.150s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 554.457956] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.457956] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62499) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 554.457956] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 6.333s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 554.459272] env[62499]: DEBUG nova.compute.manager [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Build of instance 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa was re-scheduled: Binding failed for port 8fd4fe87-772a-4544-b201-6fed68efdb0a, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 554.459871] env[62499]: DEBUG nova.compute.manager [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 554.460241] env[62499]: DEBUG oslo_concurrency.lockutils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Acquiring lock "refresh_cache-22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 554.460493] env[62499]: DEBUG oslo_concurrency.lockutils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Acquired lock "refresh_cache-22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 554.460760] env[62499]: DEBUG nova.network.neutron [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 554.462569] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ac170e5-7392-43ce-992e-17930370f378 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 554.466632] env[62499]: DEBUG oslo_concurrency.lockutils [req-429a4e0f-f655-4d7e-b666-6dde4999b12d req-4cd934e4-e232-4060-b2d7-ba13f3d7cd50 service nova] Releasing lock "refresh_cache-812c740f-eae7-4f6d-9e0d-8c16a4685a50" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 554.473899] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca347311-0759-4d8d-b15f-dcfebe7ed280 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 554.493034] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7284a4a-b3fc-4eac-86af-b145a774b172 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 554.498085] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79ad1352-cf72-4a30-b44f-576ca50cd7d4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 554.530215] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181512MB free_disk=130GB free_vcpus=48 pci_devices=None {{(pid=62499) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 554.530215] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 554.651475] env[62499]: DEBUG nova.network.neutron [-] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 554.668363] env[62499]: DEBUG nova.network.neutron [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 554.811820] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fafae4bd-23d5-4104-be21-37060a3f4aa7 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Lock "ddd212aa-9f4a-4b9d-aa60-cd4b64580f84" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 19.552s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.822923] env[62499]: DEBUG nova.network.neutron [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 554.941334] env[62499]: INFO nova.compute.manager [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] [instance: ebd5745d-f073-429c-b609-a7520a539f2b] Took 1.06 seconds to deallocate network for instance. [ 554.959636] env[62499]: DEBUG nova.compute.manager [req-4f2542e1-caac-4985-b992-8d1ba4af8745 req-8fff603f-a59d-475a-9b7a-457af19f7938 service nova] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Received event network-changed-eff92d51-a578-421d-98e9-d5ffbfb11bb1 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 554.960398] env[62499]: DEBUG nova.compute.manager [req-4f2542e1-caac-4985-b992-8d1ba4af8745 req-8fff603f-a59d-475a-9b7a-457af19f7938 service nova] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Refreshing instance network info cache due to event network-changed-eff92d51-a578-421d-98e9-d5ffbfb11bb1. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 554.960398] env[62499]: DEBUG oslo_concurrency.lockutils [req-4f2542e1-caac-4985-b992-8d1ba4af8745 req-8fff603f-a59d-475a-9b7a-457af19f7938 service nova] Acquiring lock "refresh_cache-48fc0cbf-18a6-4bbc-8455-93800b911d6a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 554.960398] env[62499]: DEBUG oslo_concurrency.lockutils [req-4f2542e1-caac-4985-b992-8d1ba4af8745 req-8fff603f-a59d-475a-9b7a-457af19f7938 service nova] Acquired lock "refresh_cache-48fc0cbf-18a6-4bbc-8455-93800b911d6a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 554.960398] env[62499]: DEBUG nova.network.neutron [req-4f2542e1-caac-4985-b992-8d1ba4af8745 req-8fff603f-a59d-475a-9b7a-457af19f7938 service nova] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Refreshing network info cache for port eff92d51-a578-421d-98e9-d5ffbfb11bb1 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 555.012550] env[62499]: DEBUG nova.network.neutron [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 555.155130] env[62499]: INFO nova.compute.manager [-] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Took 1.03 seconds to deallocate network for instance. [ 555.158636] env[62499]: DEBUG nova.compute.claims [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 555.158750] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 555.175383] env[62499]: DEBUG nova.network.neutron [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 555.220129] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ccaf8b0-bfed-4396-9f61-456c2eea55c3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 555.231378] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8cbcee8d-55fb-452a-9531-8778bce83c4a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 555.264717] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f0d9aa8-61e7-45dd-a193-cc5f65b93a4d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 555.273941] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a5e39e65-ae36-4620-97e7-fbb8f1cf76f0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 555.289842] env[62499]: DEBUG nova.compute.provider_tree [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 555.315913] env[62499]: DEBUG nova.compute.manager [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 555.324947] env[62499]: DEBUG oslo_concurrency.lockutils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Releasing lock "refresh_cache-99f77fa6-a625-4771-83c9-012f75ed1c7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 555.325293] env[62499]: DEBUG nova.compute.manager [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 555.327285] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 555.327378] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8e7fef27-44a3-4231-b1df-da8b745b8100 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 555.340186] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ffa75696-96e2-49cf-a9d0-acf72934ab15 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 555.366384] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 99f77fa6-a625-4771-83c9-012f75ed1c7d could not be found. [ 555.366384] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 555.366384] env[62499]: INFO nova.compute.manager [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Took 0.04 seconds to destroy the instance on the hypervisor. [ 555.366384] env[62499]: DEBUG oslo.service.loopingcall [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 555.366384] env[62499]: DEBUG nova.compute.manager [-] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 555.366384] env[62499]: DEBUG nova.network.neutron [-] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 555.389305] env[62499]: DEBUG nova.network.neutron [-] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 555.491185] env[62499]: DEBUG nova.network.neutron [req-4f2542e1-caac-4985-b992-8d1ba4af8745 req-8fff603f-a59d-475a-9b7a-457af19f7938 service nova] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 555.684196] env[62499]: DEBUG oslo_concurrency.lockutils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Releasing lock "refresh_cache-22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 555.684196] env[62499]: DEBUG nova.compute.manager [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 555.684196] env[62499]: DEBUG nova.compute.manager [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 555.684196] env[62499]: DEBUG nova.network.neutron [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 555.766187] env[62499]: DEBUG nova.network.neutron [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 555.793757] env[62499]: DEBUG nova.scheduler.client.report [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 555.839778] env[62499]: DEBUG nova.network.neutron [req-4f2542e1-caac-4985-b992-8d1ba4af8745 req-8fff603f-a59d-475a-9b7a-457af19f7938 service nova] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 555.847666] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 555.891433] env[62499]: DEBUG nova.network.neutron [-] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 555.915312] env[62499]: DEBUG oslo_concurrency.lockutils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "9c7433a1-9d5b-4ef2-ad65-f1217f0e862c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 555.915403] env[62499]: DEBUG oslo_concurrency.lockutils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "9c7433a1-9d5b-4ef2-ad65-f1217f0e862c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 555.978476] env[62499]: DEBUG nova.compute.manager [req-f3495506-52c9-4323-8f65-6588c2a76f42 req-2bfb2c6a-8ef0-4b4a-875b-905e272d5e1d service nova] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Received event network-vif-deleted-900edb48-3a43-497c-9834-e3a23a708ba3 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 555.978762] env[62499]: DEBUG nova.compute.manager [req-f3495506-52c9-4323-8f65-6588c2a76f42 req-2bfb2c6a-8ef0-4b4a-875b-905e272d5e1d service nova] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Received event network-changed-f71924e4-d6df-4e41-b251-783bd1c767a7 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 555.979052] env[62499]: DEBUG nova.compute.manager [req-f3495506-52c9-4323-8f65-6588c2a76f42 req-2bfb2c6a-8ef0-4b4a-875b-905e272d5e1d service nova] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Refreshing instance network info cache due to event network-changed-f71924e4-d6df-4e41-b251-783bd1c767a7. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 555.979374] env[62499]: DEBUG oslo_concurrency.lockutils [req-f3495506-52c9-4323-8f65-6588c2a76f42 req-2bfb2c6a-8ef0-4b4a-875b-905e272d5e1d service nova] Acquiring lock "refresh_cache-99f77fa6-a625-4771-83c9-012f75ed1c7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 555.979517] env[62499]: DEBUG oslo_concurrency.lockutils [req-f3495506-52c9-4323-8f65-6588c2a76f42 req-2bfb2c6a-8ef0-4b4a-875b-905e272d5e1d service nova] Acquired lock "refresh_cache-99f77fa6-a625-4771-83c9-012f75ed1c7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 555.979678] env[62499]: DEBUG nova.network.neutron [req-f3495506-52c9-4323-8f65-6588c2a76f42 req-2bfb2c6a-8ef0-4b4a-875b-905e272d5e1d service nova] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Refreshing network info cache for port f71924e4-d6df-4e41-b251-783bd1c767a7 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 555.985759] env[62499]: INFO nova.scheduler.client.report [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Deleted allocations for instance ebd5745d-f073-429c-b609-a7520a539f2b [ 556.270223] env[62499]: DEBUG nova.network.neutron [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 556.299580] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.844s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 556.300226] env[62499]: ERROR nova.compute.manager [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d253c0f7-c107-4caf-8d19-68e503db563f, please check neutron logs for more information. [ 556.300226] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Traceback (most recent call last): [ 556.300226] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 556.300226] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] self.driver.spawn(context, instance, image_meta, [ 556.300226] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 556.300226] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 556.300226] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 556.300226] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] vm_ref = self.build_virtual_machine(instance, [ 556.300226] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 556.300226] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] vif_infos = vmwarevif.get_vif_info(self._session, [ 556.300226] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 556.300626] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] for vif in network_info: [ 556.300626] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 556.300626] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] return self._sync_wrapper(fn, *args, **kwargs) [ 556.300626] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 556.300626] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] self.wait() [ 556.300626] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 556.300626] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] self[:] = self._gt.wait() [ 556.300626] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 556.300626] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] return self._exit_event.wait() [ 556.300626] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 556.300626] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] result = hub.switch() [ 556.300626] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 556.300626] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] return self.greenlet.switch() [ 556.301010] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 556.301010] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] result = function(*args, **kwargs) [ 556.301010] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 556.301010] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] return func(*args, **kwargs) [ 556.301010] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 556.301010] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] raise e [ 556.301010] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 556.301010] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] nwinfo = self.network_api.allocate_for_instance( [ 556.301010] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 556.301010] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] created_port_ids = self._update_ports_for_instance( [ 556.301010] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 556.301010] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] with excutils.save_and_reraise_exception(): [ 556.301010] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 556.301403] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] self.force_reraise() [ 556.301403] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 556.301403] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] raise self.value [ 556.301403] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 556.301403] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] updated_port = self._update_port( [ 556.301403] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 556.301403] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] _ensure_no_port_binding_failure(port) [ 556.301403] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 556.301403] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] raise exception.PortBindingFailed(port_id=port['id']) [ 556.301403] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] nova.exception.PortBindingFailed: Binding failed for port d253c0f7-c107-4caf-8d19-68e503db563f, please check neutron logs for more information. [ 556.301403] env[62499]: ERROR nova.compute.manager [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] [ 556.302704] env[62499]: DEBUG nova.compute.utils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Binding failed for port d253c0f7-c107-4caf-8d19-68e503db563f, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 556.302704] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 7.282s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 556.304117] env[62499]: INFO nova.compute.claims [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 556.308723] env[62499]: DEBUG nova.compute.manager [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Build of instance 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0 was re-scheduled: Binding failed for port d253c0f7-c107-4caf-8d19-68e503db563f, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 556.309208] env[62499]: DEBUG nova.compute.manager [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 556.309449] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Acquiring lock "refresh_cache-904c5d79-86f3-4d5b-b8ba-7e3d515b32c0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 556.309586] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Acquired lock "refresh_cache-904c5d79-86f3-4d5b-b8ba-7e3d515b32c0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 556.309740] env[62499]: DEBUG nova.network.neutron [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 556.338041] env[62499]: INFO nova.compute.manager [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Rebuilding instance [ 556.340888] env[62499]: DEBUG oslo_concurrency.lockutils [req-4f2542e1-caac-4985-b992-8d1ba4af8745 req-8fff603f-a59d-475a-9b7a-457af19f7938 service nova] Releasing lock "refresh_cache-48fc0cbf-18a6-4bbc-8455-93800b911d6a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 556.341155] env[62499]: DEBUG nova.compute.manager [req-4f2542e1-caac-4985-b992-8d1ba4af8745 req-8fff603f-a59d-475a-9b7a-457af19f7938 service nova] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Received event network-vif-deleted-eff92d51-a578-421d-98e9-d5ffbfb11bb1 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 556.398354] env[62499]: INFO nova.compute.manager [-] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Took 1.03 seconds to deallocate network for instance. [ 556.402990] env[62499]: DEBUG nova.compute.claims [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 556.404863] env[62499]: DEBUG oslo_concurrency.lockutils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 556.410855] env[62499]: DEBUG nova.compute.manager [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 556.412912] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3adae7b5-9ee2-47f7-a386-fa8760e8b245 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 556.494771] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e3eb74e0-6b45-4034-980f-c9629d2eb9b7 tempest-ServersAdminNegativeTestJSON-102187008 tempest-ServersAdminNegativeTestJSON-102187008-project-member] Lock "ebd5745d-f073-429c-b609-a7520a539f2b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 24.360s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 556.529673] env[62499]: DEBUG nova.network.neutron [req-f3495506-52c9-4323-8f65-6588c2a76f42 req-2bfb2c6a-8ef0-4b4a-875b-905e272d5e1d service nova] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 556.731863] env[62499]: DEBUG nova.network.neutron [req-f3495506-52c9-4323-8f65-6588c2a76f42 req-2bfb2c6a-8ef0-4b4a-875b-905e272d5e1d service nova] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 556.775360] env[62499]: INFO nova.compute.manager [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] [instance: 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa] Took 1.09 seconds to deallocate network for instance. [ 556.852923] env[62499]: DEBUG nova.network.neutron [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 556.925923] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Powering off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 556.926140] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-8527b77c-57ff-42de-9f05-246595d291cb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 556.937806] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 556.937806] env[62499]: value = "task-1335650" [ 556.937806] env[62499]: _type = "Task" [ 556.937806] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 556.950318] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335650, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 557.001809] env[62499]: DEBUG nova.compute.manager [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 557.049286] env[62499]: DEBUG nova.network.neutron [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 557.234495] env[62499]: DEBUG oslo_concurrency.lockutils [req-f3495506-52c9-4323-8f65-6588c2a76f42 req-2bfb2c6a-8ef0-4b4a-875b-905e272d5e1d service nova] Releasing lock "refresh_cache-99f77fa6-a625-4771-83c9-012f75ed1c7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 557.235652] env[62499]: DEBUG nova.compute.manager [req-f3495506-52c9-4323-8f65-6588c2a76f42 req-2bfb2c6a-8ef0-4b4a-875b-905e272d5e1d service nova] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Received event network-vif-deleted-f71924e4-d6df-4e41-b251-783bd1c767a7 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 557.459744] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335650, 'name': PowerOffVM_Task, 'duration_secs': 0.149567} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 557.460105] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Powered off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 557.460934] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 557.467688] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7178029b-6033-40b0-9da9-b4d65dbea477 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 557.473573] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Unregistering the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 557.473852] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-74d09843-27d0-4a86-91f0-7adeb4e4bf8f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 557.498899] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Unregistered the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 557.499133] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Deleting contents of the VM from datastore datastore2 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 557.499431] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Deleting the datastore file [datastore2] ddd212aa-9f4a-4b9d-aa60-cd4b64580f84 {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 557.499702] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-9a786d06-e4d8-4cbc-acdd-42cc9f05e113 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 557.513453] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 557.513453] env[62499]: value = "task-1335652" [ 557.513453] env[62499]: _type = "Task" [ 557.513453] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 557.524848] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335652, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 557.543365] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 557.556871] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Releasing lock "refresh_cache-904c5d79-86f3-4d5b-b8ba-7e3d515b32c0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 557.557170] env[62499]: DEBUG nova.compute.manager [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 557.557389] env[62499]: DEBUG nova.compute.manager [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 557.557582] env[62499]: DEBUG nova.network.neutron [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 557.598957] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91909abd-bb3a-475c-95d0-6f4663f8dec8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 557.609531] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2d4988b-e7eb-4267-9297-deb25d08af81 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 557.640711] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8089730-c1ea-4b45-a800-91c9950ef5f4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 557.648597] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4808af9f-2423-4155-abf6-e115d80e7a3c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 557.655393] env[62499]: DEBUG nova.network.neutron [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 557.668310] env[62499]: DEBUG nova.compute.provider_tree [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 557.828706] env[62499]: INFO nova.scheduler.client.report [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Deleted allocations for instance 22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa [ 558.024048] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335652, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.108839} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 558.024644] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Deleted the datastore file {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 558.024644] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Deleted contents of the VM from datastore datastore2 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 558.026616] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 558.084036] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Acquiring lock "0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 558.084203] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Lock "0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 558.172336] env[62499]: DEBUG nova.network.neutron [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 558.205352] env[62499]: ERROR nova.scheduler.client.report [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [req-5067f795-006d-4ea2-aa39-329ed0d7e82c] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID 38a6c941-9755-44af-8da8-6ef6281d0157. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-5067f795-006d-4ea2-aa39-329ed0d7e82c"}]} [ 558.233255] env[62499]: DEBUG nova.scheduler.client.report [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Refreshing inventories for resource provider 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:818}} [ 558.257248] env[62499]: DEBUG nova.scheduler.client.report [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Updating ProviderTree inventory for provider 38a6c941-9755-44af-8da8-6ef6281d0157 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:782}} [ 558.257631] env[62499]: DEBUG nova.compute.provider_tree [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 558.273793] env[62499]: DEBUG nova.scheduler.client.report [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Refreshing aggregate associations for resource provider 38a6c941-9755-44af-8da8-6ef6281d0157, aggregates: None {{(pid=62499) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:827}} [ 558.296923] env[62499]: DEBUG nova.scheduler.client.report [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Refreshing trait associations for resource provider 38a6c941-9755-44af-8da8-6ef6281d0157, traits: COMPUTE_NET_ATTACH_INTERFACE,HW_ARCH_X86_64,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NODE {{(pid=62499) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:839}} [ 558.337759] env[62499]: DEBUG oslo_concurrency.lockutils [None req-484bff5a-810f-4e34-ae21-8e618ef907c2 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954 tempest-FloatingIPsAssociationNegativeTestJSON-2127185954-project-member] Lock "22ff04b4-85b7-4bf3-94c4-ba4dd323d6aa" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 25.397s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 558.549386] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c5b254fc-2e3b-4421-8745-c2481b8dda3b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 558.559717] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d48b5423-d1a6-485d-8f46-9c75426a1dc7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 558.597600] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56de402f-d85c-4b2e-a5c6-86bec9362449 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 558.606179] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fdd755fb-8d03-47d7-b6c9-e97e8219d478 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 558.623900] env[62499]: DEBUG nova.compute.provider_tree [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 558.677798] env[62499]: INFO nova.compute.manager [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0] Took 1.12 seconds to deallocate network for instance. [ 558.840387] env[62499]: DEBUG nova.compute.manager [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 559.070795] env[62499]: DEBUG nova.virt.hardware [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 559.071109] env[62499]: DEBUG nova.virt.hardware [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 559.071654] env[62499]: DEBUG nova.virt.hardware [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 559.071935] env[62499]: DEBUG nova.virt.hardware [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 559.071935] env[62499]: DEBUG nova.virt.hardware [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 559.072134] env[62499]: DEBUG nova.virt.hardware [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 559.072451] env[62499]: DEBUG nova.virt.hardware [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 559.072683] env[62499]: DEBUG nova.virt.hardware [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 559.072902] env[62499]: DEBUG nova.virt.hardware [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 559.073295] env[62499]: DEBUG nova.virt.hardware [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 559.073488] env[62499]: DEBUG nova.virt.hardware [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 559.074424] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-adbaf011-2093-4995-b1a6-7dfd4845e418 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 559.086029] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c2484b7e-65c9-4e78-b2e8-eba6a38072eb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 559.102161] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Instance VIF info [] {{(pid=62499) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 559.110507] env[62499]: DEBUG oslo.service.loopingcall [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 559.110980] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Creating VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 559.111236] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-a12cea16-0397-4ab3-8f30-31df9bcae660 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 559.133084] env[62499]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 559.133084] env[62499]: value = "task-1335653" [ 559.133084] env[62499]: _type = "Task" [ 559.133084] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 559.144761] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335653, 'name': CreateVM_Task} progress is 5%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 559.174272] env[62499]: DEBUG nova.scheduler.client.report [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Updated inventory for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with generation 20 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:971}} [ 559.174528] env[62499]: DEBUG nova.compute.provider_tree [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Updating resource provider 38a6c941-9755-44af-8da8-6ef6281d0157 generation from 20 to 21 during operation: update_inventory {{(pid=62499) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 559.174970] env[62499]: DEBUG nova.compute.provider_tree [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 559.373319] env[62499]: DEBUG oslo_concurrency.lockutils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 559.643761] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335653, 'name': CreateVM_Task, 'duration_secs': 0.306424} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 559.646261] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Created VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 559.647092] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 559.647092] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquired lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 559.647277] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 559.647710] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-5e7ef812-ddb4-4fe9-a3bd-c8a270c7efe0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 559.652831] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 559.652831] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]528b0e93-0e6d-db72-4598-c28e7ba9c204" [ 559.652831] env[62499]: _type = "Task" [ 559.652831] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 559.665675] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]528b0e93-0e6d-db72-4598-c28e7ba9c204, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 559.684117] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 3.381s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 559.684666] env[62499]: DEBUG nova.compute.manager [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 559.692202] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 8.398s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 559.693704] env[62499]: INFO nova.compute.claims [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 559.764512] env[62499]: INFO nova.scheduler.client.report [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Deleted allocations for instance 904c5d79-86f3-4d5b-b8ba-7e3d515b32c0 [ 560.168419] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]528b0e93-0e6d-db72-4598-c28e7ba9c204, 'name': SearchDatastore_Task, 'duration_secs': 0.009786} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 560.168898] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Releasing lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 560.169226] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Processing image 0422123c-16ce-4c9d-b1c6-0281fe0e54ad {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 560.169426] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 560.169562] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquired lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 560.169810] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 560.170382] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-34badafe-fd9d-41da-9798-f6ac3645d971 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 560.180820] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 560.181470] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62499) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 560.182018] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-5fafc17a-50ab-4442-b66f-68e7eb243c4c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 560.189793] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 560.189793] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]5228651a-0c6a-c6d6-73af-531b7017aece" [ 560.189793] env[62499]: _type = "Task" [ 560.189793] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 560.196714] env[62499]: DEBUG nova.compute.utils [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 560.199741] env[62499]: DEBUG nova.compute.manager [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Not allocating networking since 'none' was specified. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 560.208325] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]5228651a-0c6a-c6d6-73af-531b7017aece, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 560.279550] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66269536-209c-4d6c-a3c0-6438dad90fe0 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Lock "904c5d79-86f3-4d5b-b8ba-7e3d515b32c0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 25.756s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 560.279695] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Acquiring lock "7661165d-2731-456e-af25-da13c68fd9fb" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 560.279913] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Lock "7661165d-2731-456e-af25-da13c68fd9fb" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 560.703241] env[62499]: DEBUG nova.compute.manager [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 560.705986] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]5228651a-0c6a-c6d6-73af-531b7017aece, 'name': SearchDatastore_Task, 'duration_secs': 0.020697} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 560.709075] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-3cded046-697e-4a3f-9360-33e18ba08287 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 560.717948] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 560.717948] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]526dc373-9720-0ef7-972a-930444b61f6d" [ 560.717948] env[62499]: _type = "Task" [ 560.717948] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 560.741421] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]526dc373-9720-0ef7-972a-930444b61f6d, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 560.783228] env[62499]: DEBUG nova.compute.manager [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 560.976904] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e005572-239f-4bb9-bad3-a54ff89cf44d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 560.984789] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a465ad4a-620d-4ba4-b29a-5ff67153e605 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.016088] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1102fe6-8794-4ed4-8760-2752a18d676a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.023555] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd586b85-66ab-4e23-9928-113b50f436ab {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.038044] env[62499]: DEBUG nova.compute.provider_tree [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 561.229505] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]526dc373-9720-0ef7-972a-930444b61f6d, 'name': SearchDatastore_Task, 'duration_secs': 0.016062} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 561.229505] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Releasing lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 561.229878] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore2] ddd212aa-9f4a-4b9d-aa60-cd4b64580f84/ddd212aa-9f4a-4b9d-aa60-cd4b64580f84.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 561.230020] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-8f003516-0151-44f8-b587-6d2378f385c2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.239656] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 561.239656] env[62499]: value = "task-1335654" [ 561.239656] env[62499]: _type = "Task" [ 561.239656] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 561.250507] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335654, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 561.314811] env[62499]: DEBUG oslo_concurrency.lockutils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 561.543173] env[62499]: DEBUG nova.scheduler.client.report [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 561.715153] env[62499]: DEBUG nova.compute.manager [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 561.752195] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335654, 'name': CopyVirtualDisk_Task} progress is 25%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 561.755116] env[62499]: DEBUG nova.virt.hardware [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 561.755341] env[62499]: DEBUG nova.virt.hardware [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 561.755505] env[62499]: DEBUG nova.virt.hardware [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 561.755678] env[62499]: DEBUG nova.virt.hardware [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 561.755817] env[62499]: DEBUG nova.virt.hardware [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 561.755980] env[62499]: DEBUG nova.virt.hardware [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 561.757082] env[62499]: DEBUG nova.virt.hardware [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 561.757082] env[62499]: DEBUG nova.virt.hardware [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 561.757082] env[62499]: DEBUG nova.virt.hardware [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 561.757082] env[62499]: DEBUG nova.virt.hardware [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 561.757082] env[62499]: DEBUG nova.virt.hardware [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 561.757832] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3206d07-947f-4553-be28-015865f3dcc1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.770083] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad759bd6-071a-4149-b752-f4f322ea74be {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.789592] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Instance VIF info [] {{(pid=62499) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 561.797441] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Creating folder: Project (55d5905bb740438f87ef1bd846cbcb23). Parent ref: group-v285191. {{(pid=62499) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 561.797836] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-c21887d5-db90-4610-9ff4-1810e51aae2e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.810104] env[62499]: INFO nova.virt.vmwareapi.vm_util [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Created folder: Project (55d5905bb740438f87ef1bd846cbcb23) in parent group-v285191. [ 561.810470] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Creating folder: Instances. Parent ref: group-v285196. {{(pid=62499) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 561.810648] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-68004003-cf4b-4623-b867-1d6b715ae47a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.821572] env[62499]: INFO nova.virt.vmwareapi.vm_util [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Created folder: Instances in parent group-v285196. [ 561.821851] env[62499]: DEBUG oslo.service.loopingcall [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 561.822167] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Creating VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 561.822324] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-2ecb0a83-4ab4-496c-8a37-33f9937e9eba {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.841708] env[62499]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 561.841708] env[62499]: value = "task-1335657" [ 561.841708] env[62499]: _type = "Task" [ 561.841708] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 561.853451] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335657, 'name': CreateVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 562.054206] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.362s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 562.055019] env[62499]: DEBUG nova.compute.manager [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 562.059164] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 9.477s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 562.060721] env[62499]: INFO nova.compute.claims [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 562.257299] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335654, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.701238} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 562.258630] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore2] ddd212aa-9f4a-4b9d-aa60-cd4b64580f84/ddd212aa-9f4a-4b9d-aa60-cd4b64580f84.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 562.260736] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Extending root virtual disk to 1048576 {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 562.260736] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-b5e86633-3bad-4730-9982-f28716c9fd8b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.267094] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 562.267094] env[62499]: value = "task-1335658" [ 562.267094] env[62499]: _type = "Task" [ 562.267094] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 562.280334] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335658, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 562.353313] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335657, 'name': CreateVM_Task, 'duration_secs': 0.400168} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 562.353565] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Created VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 562.354217] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 562.355966] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Acquired lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 562.356047] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 562.356329] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-e2c806e7-3623-45b4-bae4-8a11fec7e3e0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.361969] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Waiting for the task: (returnval){ [ 562.361969] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52ef6363-3af0-4a20-796c-3ba342ab6070" [ 562.361969] env[62499]: _type = "Task" [ 562.361969] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 562.375527] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52ef6363-3af0-4a20-796c-3ba342ab6070, 'name': SearchDatastore_Task, 'duration_secs': 0.008771} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 562.375812] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Releasing lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 562.376053] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Processing image 0422123c-16ce-4c9d-b1c6-0281fe0e54ad {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 562.376277] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 562.376420] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Acquired lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 562.376591] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 562.376848] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-f5a487c1-0d86-4cd7-9ee2-fa69d301ee16 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.390096] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 562.390295] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62499) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 562.391055] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-69affdba-bc44-457d-9e06-ffcd6ea4d58b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.401143] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Waiting for the task: (returnval){ [ 562.401143] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52ddc58f-a2ce-c0b5-bffb-ed82bab1ad05" [ 562.401143] env[62499]: _type = "Task" [ 562.401143] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 562.409857] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52ddc58f-a2ce-c0b5-bffb-ed82bab1ad05, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 562.566851] env[62499]: DEBUG nova.compute.utils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 562.573595] env[62499]: DEBUG nova.compute.manager [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 562.573595] env[62499]: DEBUG nova.network.neutron [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 562.619738] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Acquiring lock "037217df-a3ed-4cef-9478-a9cc576a2f7d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 562.619970] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Lock "037217df-a3ed-4cef-9478-a9cc576a2f7d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 562.776754] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335658, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.066248} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 562.777071] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Extended root virtual disk {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 562.777936] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc502bf9-014e-40b2-b3c1-e3b5968af214 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.799035] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Reconfiguring VM instance instance-00000004 to attach disk [datastore2] ddd212aa-9f4a-4b9d-aa60-cd4b64580f84/ddd212aa-9f4a-4b9d-aa60-cd4b64580f84.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 562.799421] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-c43eba93-59cf-4838-98af-e6f12f1ae55d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.821793] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 562.821793] env[62499]: value = "task-1335659" [ 562.821793] env[62499]: _type = "Task" [ 562.821793] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 562.834365] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335659, 'name': ReconfigVM_Task} progress is 10%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 562.865465] env[62499]: DEBUG nova.policy [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a4763a330d70462182e5bdccc2a08c25', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9512415a4f124e8c9120f6c115f78949', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 562.912991] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52ddc58f-a2ce-c0b5-bffb-ed82bab1ad05, 'name': SearchDatastore_Task, 'duration_secs': 0.008955} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 562.915426] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-2b7cb9b5-87e1-43c8-9a32-4d27a15f740d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 562.921818] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Waiting for the task: (returnval){ [ 562.921818] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]523514c3-a263-8986-bb58-61df3d586842" [ 562.921818] env[62499]: _type = "Task" [ 562.921818] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 562.932776] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]523514c3-a263-8986-bb58-61df3d586842, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 563.075531] env[62499]: DEBUG nova.compute.manager [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 563.339627] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335659, 'name': ReconfigVM_Task, 'duration_secs': 0.271103} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 563.339627] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Reconfigured VM instance instance-00000004 to attach disk [datastore2] ddd212aa-9f4a-4b9d-aa60-cd4b64580f84/ddd212aa-9f4a-4b9d-aa60-cd4b64580f84.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 563.339627] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-584d3097-75b9-4cc1-a322-ee09fa1f643d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 563.350675] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 563.350675] env[62499]: value = "task-1335660" [ 563.350675] env[62499]: _type = "Task" [ 563.350675] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 563.361786] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335660, 'name': Rename_Task} progress is 6%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 563.380516] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e696af81-a583-4d1e-bca3-9c3714c45b15 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 563.391220] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84221b4b-8b8e-4751-aeee-c9cf60330ca4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 563.431981] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d75fa5f-e31d-44ed-9b40-17052fd93100 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 563.441124] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]523514c3-a263-8986-bb58-61df3d586842, 'name': SearchDatastore_Task, 'duration_secs': 0.008647} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 563.444885] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Releasing lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 563.444885] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore2] 27103ce2-0962-4fce-9331-d74179b0510c/27103ce2-0962-4fce-9331-d74179b0510c.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 563.444885] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-7e594ba1-32d7-465f-8665-bbc1ab50a5d6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 563.447013] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67bee13f-b7db-435e-b46a-07cb079ea49f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 563.463074] env[62499]: DEBUG nova.compute.provider_tree [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 563.465947] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Waiting for the task: (returnval){ [ 563.465947] env[62499]: value = "task-1335661" [ 563.465947] env[62499]: _type = "Task" [ 563.465947] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 563.474727] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': task-1335661, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 563.864022] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335660, 'name': Rename_Task, 'duration_secs': 0.125273} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 563.864022] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Powering on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 563.864350] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-6c5da31f-e2c0-4423-a75b-b50e767f60c1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 563.871355] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 563.871355] env[62499]: value = "task-1335662" [ 563.871355] env[62499]: _type = "Task" [ 563.871355] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 563.882606] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335662, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 563.985066] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': task-1335661, 'name': CopyVirtualDisk_Task} progress is 77%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 563.999462] env[62499]: ERROR nova.scheduler.client.report [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [req-9a05bdea-17e4-4526-84c2-5e687fbf978f] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID 38a6c941-9755-44af-8da8-6ef6281d0157. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-9a05bdea-17e4-4526-84c2-5e687fbf978f"}]} [ 564.038473] env[62499]: DEBUG nova.scheduler.client.report [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Refreshing inventories for resource provider 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:818}} [ 564.064816] env[62499]: DEBUG nova.scheduler.client.report [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Updating ProviderTree inventory for provider 38a6c941-9755-44af-8da8-6ef6281d0157 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:782}} [ 564.065423] env[62499]: DEBUG nova.compute.provider_tree [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 131, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 564.081548] env[62499]: DEBUG nova.scheduler.client.report [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Refreshing aggregate associations for resource provider 38a6c941-9755-44af-8da8-6ef6281d0157, aggregates: None {{(pid=62499) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:827}} [ 564.087947] env[62499]: DEBUG nova.compute.manager [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 564.116613] env[62499]: DEBUG nova.scheduler.client.report [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Refreshing trait associations for resource provider 38a6c941-9755-44af-8da8-6ef6281d0157, traits: COMPUTE_NET_ATTACH_INTERFACE,HW_ARCH_X86_64,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NODE {{(pid=62499) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:839}} [ 564.143996] env[62499]: DEBUG nova.virt.hardware [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 564.143996] env[62499]: DEBUG nova.virt.hardware [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 564.143996] env[62499]: DEBUG nova.virt.hardware [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 564.144414] env[62499]: DEBUG nova.virt.hardware [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 564.144414] env[62499]: DEBUG nova.virt.hardware [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 564.144414] env[62499]: DEBUG nova.virt.hardware [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 564.144414] env[62499]: DEBUG nova.virt.hardware [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 564.144414] env[62499]: DEBUG nova.virt.hardware [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 564.144577] env[62499]: DEBUG nova.virt.hardware [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 564.144577] env[62499]: DEBUG nova.virt.hardware [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 564.144846] env[62499]: DEBUG nova.virt.hardware [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 564.145764] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea953b6d-d5f8-4150-ab8d-33b30cf3c679 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.155265] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-625db420-8c40-4839-98d5-266fcc017440 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.390913] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335662, 'name': PowerOnVM_Task} progress is 66%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 564.448116] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cfd41d5a-5e59-42ec-a80c-630828917154 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.459228] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02ab7989-3ca6-4ef5-9993-745250b90032 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.498069] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-73736d6b-ef50-4f6a-84ef-3ac230728b88 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.506947] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': task-1335661, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.589153} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 564.509398] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore2] 27103ce2-0962-4fce-9331-d74179b0510c/27103ce2-0962-4fce-9331-d74179b0510c.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 564.510062] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Extending root virtual disk to 1048576 {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 564.510062] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-144fa3e9-1d3c-46a8-9fbf-0d8e1cc40e60 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.512763] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ca30eca-5e4c-4d1c-9ca6-588b9e14354e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.533441] env[62499]: DEBUG nova.compute.provider_tree [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 564.536881] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Waiting for the task: (returnval){ [ 564.536881] env[62499]: value = "task-1335663" [ 564.536881] env[62499]: _type = "Task" [ 564.536881] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 564.544142] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': task-1335663, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 564.886178] env[62499]: DEBUG oslo_vmware.api [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335662, 'name': PowerOnVM_Task, 'duration_secs': 0.694005} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 564.886788] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Powered on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 564.887059] env[62499]: DEBUG nova.compute.manager [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 564.887914] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e1156833-8336-4613-9160-ae158ab87a10 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.926780] env[62499]: DEBUG nova.network.neutron [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Successfully created port: 42bcc024-8460-4983-9bc4-d3545aa63650 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 565.052425] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': task-1335663, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.06887} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 565.052425] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Extended root virtual disk {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 565.053378] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a450ddf1-2bd5-4d1c-8f87-dec70160c9f4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 565.074834] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Reconfiguring VM instance instance-00000008 to attach disk [datastore2] 27103ce2-0962-4fce-9331-d74179b0510c/27103ce2-0962-4fce-9331-d74179b0510c.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 565.075202] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-8b568a7b-eb44-4f02-a6b5-f3b5b69daeda {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 565.092236] env[62499]: DEBUG nova.scheduler.client.report [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Updated inventory for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with generation 23 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:971}} [ 565.092466] env[62499]: DEBUG nova.compute.provider_tree [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Updating resource provider 38a6c941-9755-44af-8da8-6ef6281d0157 generation from 23 to 24 during operation: update_inventory {{(pid=62499) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 565.092639] env[62499]: DEBUG nova.compute.provider_tree [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 565.101862] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Waiting for the task: (returnval){ [ 565.101862] env[62499]: value = "task-1335664" [ 565.101862] env[62499]: _type = "Task" [ 565.101862] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 565.111876] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': task-1335664, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 565.407232] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 565.602764] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 3.542s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 565.602764] env[62499]: DEBUG nova.compute.manager [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 565.607825] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.280s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 565.621295] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': task-1335664, 'name': ReconfigVM_Task, 'duration_secs': 0.413831} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 565.621689] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Reconfigured VM instance instance-00000008 to attach disk [datastore2] 27103ce2-0962-4fce-9331-d74179b0510c/27103ce2-0962-4fce-9331-d74179b0510c.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 565.622483] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-a38df02c-797e-4f69-97e2-3f9af7a9586b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 565.630221] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Waiting for the task: (returnval){ [ 565.630221] env[62499]: value = "task-1335665" [ 565.630221] env[62499]: _type = "Task" [ 565.630221] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 565.644719] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': task-1335665, 'name': Rename_Task} progress is 5%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 566.112877] env[62499]: DEBUG nova.compute.utils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 566.114123] env[62499]: DEBUG nova.compute.manager [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 566.114311] env[62499]: DEBUG nova.network.neutron [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 566.142074] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': task-1335665, 'name': Rename_Task, 'duration_secs': 0.133448} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 566.142074] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Powering on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 566.142502] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-dce25509-78b0-4905-b914-d24b67a02117 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 566.150857] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Waiting for the task: (returnval){ [ 566.150857] env[62499]: value = "task-1335666" [ 566.150857] env[62499]: _type = "Task" [ 566.150857] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 566.159122] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': task-1335666, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 566.345990] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52831f30-a5f5-43ff-bc4b-e1beb78d77ae {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 566.357330] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5cf69c6-680f-45da-b42c-b8f6c210a6f0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 566.394768] env[62499]: DEBUG nova.policy [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3b351c419e664efd8066ab086962877d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'b7d6c1f825564159823ed1b5e42e6b02', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 566.396977] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-787a300f-1a7a-44a7-b119-e580100ae956 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 566.405142] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-61313a79-c82d-4125-bbd8-779bede44942 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 566.420351] env[62499]: DEBUG nova.compute.provider_tree [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 566.618572] env[62499]: DEBUG nova.compute.manager [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 566.659799] env[62499]: DEBUG oslo_vmware.api [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': task-1335666, 'name': PowerOnVM_Task, 'duration_secs': 0.454099} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 566.662222] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Powered on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 566.662222] env[62499]: INFO nova.compute.manager [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Took 4.95 seconds to spawn the instance on the hypervisor. [ 566.662222] env[62499]: DEBUG nova.compute.manager [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 566.662935] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e52d8b64-7577-4b65-96f6-8d480f82cab5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 566.874462] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Acquiring lock "9160ca3a-2aa2-4cfc-9900-3e4c770d2657" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 566.874706] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Lock "9160ca3a-2aa2-4cfc-9900-3e4c770d2657" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 566.923329] env[62499]: DEBUG nova.scheduler.client.report [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 567.190120] env[62499]: INFO nova.compute.manager [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Took 18.20 seconds to build instance. [ 567.429276] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.825s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 567.429961] env[62499]: ERROR nova.compute.manager [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 900edb48-3a43-497c-9834-e3a23a708ba3, please check neutron logs for more information. [ 567.429961] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Traceback (most recent call last): [ 567.429961] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 567.429961] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] self.driver.spawn(context, instance, image_meta, [ 567.429961] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 567.429961] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] self._vmops.spawn(context, instance, image_meta, injected_files, [ 567.429961] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 567.429961] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] vm_ref = self.build_virtual_machine(instance, [ 567.429961] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 567.429961] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] vif_infos = vmwarevif.get_vif_info(self._session, [ 567.429961] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 567.430520] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] for vif in network_info: [ 567.430520] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 567.430520] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] return self._sync_wrapper(fn, *args, **kwargs) [ 567.430520] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 567.430520] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] self.wait() [ 567.430520] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 567.430520] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] self[:] = self._gt.wait() [ 567.430520] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 567.430520] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] return self._exit_event.wait() [ 567.430520] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 567.430520] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] result = hub.switch() [ 567.430520] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 567.430520] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] return self.greenlet.switch() [ 567.431055] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 567.431055] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] result = function(*args, **kwargs) [ 567.431055] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 567.431055] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] return func(*args, **kwargs) [ 567.431055] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 567.431055] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] raise e [ 567.431055] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 567.431055] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] nwinfo = self.network_api.allocate_for_instance( [ 567.431055] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 567.431055] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] created_port_ids = self._update_ports_for_instance( [ 567.431055] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 567.431055] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] with excutils.save_and_reraise_exception(): [ 567.431055] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 567.431595] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] self.force_reraise() [ 567.431595] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 567.431595] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] raise self.value [ 567.431595] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 567.431595] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] updated_port = self._update_port( [ 567.431595] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 567.431595] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] _ensure_no_port_binding_failure(port) [ 567.431595] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 567.431595] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] raise exception.PortBindingFailed(port_id=port['id']) [ 567.431595] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] nova.exception.PortBindingFailed: Binding failed for port 900edb48-3a43-497c-9834-e3a23a708ba3, please check neutron logs for more information. [ 567.431595] env[62499]: ERROR nova.compute.manager [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] [ 567.432028] env[62499]: DEBUG nova.compute.utils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Binding failed for port 900edb48-3a43-497c-9834-e3a23a708ba3, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 567.432028] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 12.902s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 567.436714] env[62499]: DEBUG nova.compute.manager [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Build of instance 812c740f-eae7-4f6d-9e0d-8c16a4685a50 was re-scheduled: Binding failed for port 900edb48-3a43-497c-9834-e3a23a708ba3, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 567.437208] env[62499]: DEBUG nova.compute.manager [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 567.437449] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Acquiring lock "refresh_cache-812c740f-eae7-4f6d-9e0d-8c16a4685a50" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 567.437622] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Acquired lock "refresh_cache-812c740f-eae7-4f6d-9e0d-8c16a4685a50" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 567.437795] env[62499]: DEBUG nova.network.neutron [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 567.639282] env[62499]: DEBUG nova.compute.manager [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 567.673086] env[62499]: DEBUG nova.virt.hardware [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 567.673563] env[62499]: DEBUG nova.virt.hardware [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 567.673790] env[62499]: DEBUG nova.virt.hardware [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 567.674096] env[62499]: DEBUG nova.virt.hardware [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 567.674421] env[62499]: DEBUG nova.virt.hardware [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 567.674662] env[62499]: DEBUG nova.virt.hardware [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 567.674892] env[62499]: DEBUG nova.virt.hardware [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 567.675065] env[62499]: DEBUG nova.virt.hardware [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 567.675238] env[62499]: DEBUG nova.virt.hardware [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 567.675392] env[62499]: DEBUG nova.virt.hardware [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 567.675647] env[62499]: DEBUG nova.virt.hardware [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 567.676556] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42c1975e-c2b1-42ca-807f-86384d77ec72 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 567.685710] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-663a685a-498f-4fa1-8a14-d22f8300b042 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 567.691382] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ef1e9478-3440-44ab-9c16-bb88009b525b tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Lock "27103ce2-0962-4fce-9331-d74179b0510c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 19.707s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 567.998875] env[62499]: DEBUG nova.network.neutron [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Successfully created port: 60f230d6-abe8-45a2-896b-081db4709502 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 568.194025] env[62499]: DEBUG nova.compute.manager [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 568.260038] env[62499]: DEBUG nova.network.neutron [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 568.493999] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance ddd212aa-9f4a-4b9d-aa60-cd4b64580f84 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 568.557439] env[62499]: DEBUG nova.network.neutron [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 568.726094] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 568.998921] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 812c740f-eae7-4f6d-9e0d-8c16a4685a50 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 569.001852] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 48fc0cbf-18a6-4bbc-8455-93800b911d6a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 569.001852] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 99f77fa6-a625-4771-83c9-012f75ed1c7d actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 569.001852] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 27103ce2-0962-4fce-9331-d74179b0510c actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 569.001852] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 2b7122b2-136d-4e0c-97eb-155b33905436 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 569.002142] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 76cf7f28-7413-4b06-bd30-ae22bbc2999e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 569.060952] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Releasing lock "refresh_cache-812c740f-eae7-4f6d-9e0d-8c16a4685a50" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 569.061061] env[62499]: DEBUG nova.compute.manager [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 569.061347] env[62499]: DEBUG nova.compute.manager [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 569.061420] env[62499]: DEBUG nova.network.neutron [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 569.144811] env[62499]: DEBUG nova.network.neutron [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 569.506617] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 8d45497d-ba77-45b1-91f3-719b2689bee3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 569.617253] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Acquiring lock "81b1d279-dfad-4a2c-bcfb-043b13d77af6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 569.617700] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Lock "81b1d279-dfad-4a2c-bcfb-043b13d77af6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 569.650939] env[62499]: DEBUG nova.network.neutron [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 569.834390] env[62499]: ERROR nova.compute.manager [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 42bcc024-8460-4983-9bc4-d3545aa63650, please check neutron logs for more information. [ 569.834390] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 569.834390] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 569.834390] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 569.834390] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 569.834390] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 569.834390] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 569.834390] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 569.834390] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 569.834390] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 569.834390] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 569.834390] env[62499]: ERROR nova.compute.manager raise self.value [ 569.834390] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 569.834390] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 569.834390] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 569.834390] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 569.835309] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 569.835309] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 569.835309] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 42bcc024-8460-4983-9bc4-d3545aa63650, please check neutron logs for more information. [ 569.835309] env[62499]: ERROR nova.compute.manager [ 569.835309] env[62499]: Traceback (most recent call last): [ 569.835309] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 569.835309] env[62499]: listener.cb(fileno) [ 569.835309] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 569.835309] env[62499]: result = function(*args, **kwargs) [ 569.835309] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 569.835309] env[62499]: return func(*args, **kwargs) [ 569.835309] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 569.835309] env[62499]: raise e [ 569.835309] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 569.835309] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 569.835309] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 569.835309] env[62499]: created_port_ids = self._update_ports_for_instance( [ 569.835309] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 569.835309] env[62499]: with excutils.save_and_reraise_exception(): [ 569.835309] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 569.835309] env[62499]: self.force_reraise() [ 569.835309] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 569.835309] env[62499]: raise self.value [ 569.835309] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 569.835309] env[62499]: updated_port = self._update_port( [ 569.835309] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 569.835309] env[62499]: _ensure_no_port_binding_failure(port) [ 569.835309] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 569.835309] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 569.836245] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 42bcc024-8460-4983-9bc4-d3545aa63650, please check neutron logs for more information. [ 569.836245] env[62499]: Removing descriptor: 16 [ 569.836245] env[62499]: ERROR nova.compute.manager [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 42bcc024-8460-4983-9bc4-d3545aa63650, please check neutron logs for more information. [ 569.836245] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Traceback (most recent call last): [ 569.836245] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 569.836245] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] yield resources [ 569.836245] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 569.836245] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] self.driver.spawn(context, instance, image_meta, [ 569.836245] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 569.836245] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] self._vmops.spawn(context, instance, image_meta, injected_files, [ 569.836245] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 569.836245] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] vm_ref = self.build_virtual_machine(instance, [ 569.836636] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 569.836636] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] vif_infos = vmwarevif.get_vif_info(self._session, [ 569.836636] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 569.836636] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] for vif in network_info: [ 569.836636] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 569.836636] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] return self._sync_wrapper(fn, *args, **kwargs) [ 569.836636] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 569.836636] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] self.wait() [ 569.836636] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 569.836636] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] self[:] = self._gt.wait() [ 569.836636] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 569.836636] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] return self._exit_event.wait() [ 569.836636] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 569.837024] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] result = hub.switch() [ 569.837024] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 569.837024] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] return self.greenlet.switch() [ 569.837024] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 569.837024] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] result = function(*args, **kwargs) [ 569.837024] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 569.837024] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] return func(*args, **kwargs) [ 569.837024] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 569.837024] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] raise e [ 569.837024] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 569.837024] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] nwinfo = self.network_api.allocate_for_instance( [ 569.837024] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 569.837024] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] created_port_ids = self._update_ports_for_instance( [ 569.837425] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 569.837425] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] with excutils.save_and_reraise_exception(): [ 569.837425] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 569.837425] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] self.force_reraise() [ 569.837425] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 569.837425] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] raise self.value [ 569.837425] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 569.837425] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] updated_port = self._update_port( [ 569.837425] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 569.837425] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] _ensure_no_port_binding_failure(port) [ 569.837425] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 569.837425] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] raise exception.PortBindingFailed(port_id=port['id']) [ 569.838803] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] nova.exception.PortBindingFailed: Binding failed for port 42bcc024-8460-4983-9bc4-d3545aa63650, please check neutron logs for more information. [ 569.838803] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] [ 569.838803] env[62499]: INFO nova.compute.manager [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Terminating instance [ 569.839222] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquiring lock "refresh_cache-2b7122b2-136d-4e0c-97eb-155b33905436" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 569.839362] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquired lock "refresh_cache-2b7122b2-136d-4e0c-97eb-155b33905436" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 569.839506] env[62499]: DEBUG nova.network.neutron [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 570.011598] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 570.113356] env[62499]: INFO nova.compute.manager [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Rebuilding instance [ 570.152394] env[62499]: INFO nova.compute.manager [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] [instance: 812c740f-eae7-4f6d-9e0d-8c16a4685a50] Took 1.09 seconds to deallocate network for instance. [ 570.173161] env[62499]: DEBUG nova.compute.manager [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 570.177324] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ace4f922-e7db-4c26-8bbb-0f6a0ec0bbd5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 570.430797] env[62499]: DEBUG nova.network.neutron [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 570.515734] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 3706243f-4768-4b44-9a5b-730bf2b18974 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 570.686210] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Powering off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 570.686537] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-e6526951-80b5-429a-9369-856d05dd6072 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 570.694072] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Waiting for the task: (returnval){ [ 570.694072] env[62499]: value = "task-1335667" [ 570.694072] env[62499]: _type = "Task" [ 570.694072] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 570.705090] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335667, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 570.776653] env[62499]: DEBUG nova.compute.manager [None req-88c2483a-1f4c-4fbf-bcb2-f170034ce586 tempest-ServerDiagnosticsV248Test-1831854974 tempest-ServerDiagnosticsV248Test-1831854974-project-admin] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 570.778808] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bb9003ec-9aa8-4540-b719-ecebf49941c3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 570.787760] env[62499]: INFO nova.compute.manager [None req-88c2483a-1f4c-4fbf-bcb2-f170034ce586 tempest-ServerDiagnosticsV248Test-1831854974 tempest-ServerDiagnosticsV248Test-1831854974-project-admin] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Retrieving diagnostics [ 570.789316] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3c2c5185-52ca-4eb8-a141-f0c5cfc7aa7a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 570.794381] env[62499]: DEBUG nova.network.neutron [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 570.831895] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Releasing lock "refresh_cache-2b7122b2-136d-4e0c-97eb-155b33905436" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 570.831895] env[62499]: DEBUG nova.compute.manager [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 570.831895] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 570.832119] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ec04acf9-ed20-48b2-85dd-e1e3ff30819d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 570.842384] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32a8f3ed-75ce-4adf-9854-dd79bb5d8641 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 570.871236] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 2b7122b2-136d-4e0c-97eb-155b33905436 could not be found. [ 570.871590] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 570.871689] env[62499]: INFO nova.compute.manager [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Took 0.04 seconds to destroy the instance on the hypervisor. [ 570.871948] env[62499]: DEBUG oslo.service.loopingcall [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 570.872599] env[62499]: DEBUG nova.compute.manager [-] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 570.872711] env[62499]: DEBUG nova.network.neutron [-] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 570.928496] env[62499]: DEBUG nova.network.neutron [-] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 571.020910] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 571.042023] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Acquiring lock "ccce765d-8df9-4837-b166-71e41bea6de7" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 571.044341] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Lock "ccce765d-8df9-4837-b166-71e41bea6de7" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 571.212140] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335667, 'name': PowerOffVM_Task, 'duration_secs': 0.123959} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 571.212140] env[62499]: INFO nova.scheduler.client.report [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Deleted allocations for instance 812c740f-eae7-4f6d-9e0d-8c16a4685a50 [ 571.216283] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Powered off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 571.216523] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 571.218361] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28783118-38ba-4531-b08e-b5938c0c28c5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 571.226100] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Unregistering the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 571.226411] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-7d190964-50b8-46f6-a421-de9074d19f9c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 571.250974] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Unregistered the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 571.250974] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Deleting contents of the VM from datastore datastore2 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 571.250974] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Deleting the datastore file [datastore2] ddd212aa-9f4a-4b9d-aa60-cd4b64580f84 {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 571.250974] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-53353525-f3f3-410e-a878-c309ca1f1912 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 571.258640] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Waiting for the task: (returnval){ [ 571.258640] env[62499]: value = "task-1335669" [ 571.258640] env[62499]: _type = "Task" [ 571.258640] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 571.266446] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335669, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 571.432155] env[62499]: DEBUG nova.network.neutron [-] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 571.525041] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 571.723246] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22f8f427-be8b-4f44-b491-2b7a8689330a tempest-ServerDiagnosticsTest-514470568 tempest-ServerDiagnosticsTest-514470568-project-member] Lock "812c740f-eae7-4f6d-9e0d-8c16a4685a50" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 35.884s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 571.774040] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335669, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.092999} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 571.774040] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Deleted the datastore file {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 571.774040] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Deleted contents of the VM from datastore datastore2 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 571.774040] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 571.938585] env[62499]: INFO nova.compute.manager [-] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Took 1.07 seconds to deallocate network for instance. [ 571.941754] env[62499]: DEBUG nova.compute.claims [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 571.941906] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 572.007983] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "287c3404-9fc1-4369-b9a4-17da918bf78d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 572.007983] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "287c3404-9fc1-4369-b9a4-17da918bf78d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 572.030522] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 7661165d-2731-456e-af25-da13c68fd9fb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 572.042324] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "6680c707-aa59-4e00-a2b1-38d465cc6497" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 572.042656] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "6680c707-aa59-4e00-a2b1-38d465cc6497" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 572.068170] env[62499]: DEBUG nova.compute.manager [req-cc7d185b-dde7-49aa-8358-e66b73a13ad2 req-0d240bbb-93e9-4427-a73a-25fc63222156 service nova] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Received event network-changed-42bcc024-8460-4983-9bc4-d3545aa63650 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 572.068363] env[62499]: DEBUG nova.compute.manager [req-cc7d185b-dde7-49aa-8358-e66b73a13ad2 req-0d240bbb-93e9-4427-a73a-25fc63222156 service nova] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Refreshing instance network info cache due to event network-changed-42bcc024-8460-4983-9bc4-d3545aa63650. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 572.068581] env[62499]: DEBUG oslo_concurrency.lockutils [req-cc7d185b-dde7-49aa-8358-e66b73a13ad2 req-0d240bbb-93e9-4427-a73a-25fc63222156 service nova] Acquiring lock "refresh_cache-2b7122b2-136d-4e0c-97eb-155b33905436" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 572.068720] env[62499]: DEBUG oslo_concurrency.lockutils [req-cc7d185b-dde7-49aa-8358-e66b73a13ad2 req-0d240bbb-93e9-4427-a73a-25fc63222156 service nova] Acquired lock "refresh_cache-2b7122b2-136d-4e0c-97eb-155b33905436" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 572.069881] env[62499]: DEBUG nova.network.neutron [req-cc7d185b-dde7-49aa-8358-e66b73a13ad2 req-0d240bbb-93e9-4427-a73a-25fc63222156 service nova] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Refreshing network info cache for port 42bcc024-8460-4983-9bc4-d3545aa63650 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 572.229717] env[62499]: DEBUG nova.compute.manager [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 572.533184] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 037217df-a3ed-4cef-9478-a9cc576a2f7d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 572.613716] env[62499]: DEBUG nova.network.neutron [req-cc7d185b-dde7-49aa-8358-e66b73a13ad2 req-0d240bbb-93e9-4427-a73a-25fc63222156 service nova] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 572.695121] env[62499]: ERROR nova.compute.manager [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 60f230d6-abe8-45a2-896b-081db4709502, please check neutron logs for more information. [ 572.695121] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 572.695121] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 572.695121] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 572.695121] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 572.695121] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 572.695121] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 572.695121] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 572.695121] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 572.695121] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 572.695121] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 572.695121] env[62499]: ERROR nova.compute.manager raise self.value [ 572.695121] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 572.695121] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 572.695121] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 572.695121] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 572.695732] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 572.695732] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 572.695732] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 60f230d6-abe8-45a2-896b-081db4709502, please check neutron logs for more information. [ 572.695732] env[62499]: ERROR nova.compute.manager [ 572.695732] env[62499]: Traceback (most recent call last): [ 572.695732] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 572.695732] env[62499]: listener.cb(fileno) [ 572.695732] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 572.695732] env[62499]: result = function(*args, **kwargs) [ 572.695732] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 572.695732] env[62499]: return func(*args, **kwargs) [ 572.695732] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 572.695732] env[62499]: raise e [ 572.695732] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 572.695732] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 572.695732] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 572.695732] env[62499]: created_port_ids = self._update_ports_for_instance( [ 572.695732] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 572.695732] env[62499]: with excutils.save_and_reraise_exception(): [ 572.695732] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 572.695732] env[62499]: self.force_reraise() [ 572.695732] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 572.695732] env[62499]: raise self.value [ 572.695732] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 572.695732] env[62499]: updated_port = self._update_port( [ 572.695732] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 572.695732] env[62499]: _ensure_no_port_binding_failure(port) [ 572.695732] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 572.695732] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 572.697152] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 60f230d6-abe8-45a2-896b-081db4709502, please check neutron logs for more information. [ 572.697152] env[62499]: Removing descriptor: 15 [ 572.697152] env[62499]: ERROR nova.compute.manager [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 60f230d6-abe8-45a2-896b-081db4709502, please check neutron logs for more information. [ 572.697152] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Traceback (most recent call last): [ 572.697152] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 572.697152] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] yield resources [ 572.697152] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 572.697152] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] self.driver.spawn(context, instance, image_meta, [ 572.697152] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 572.697152] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 572.697152] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 572.697152] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] vm_ref = self.build_virtual_machine(instance, [ 572.697864] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 572.697864] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] vif_infos = vmwarevif.get_vif_info(self._session, [ 572.697864] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 572.697864] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] for vif in network_info: [ 572.697864] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 572.697864] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] return self._sync_wrapper(fn, *args, **kwargs) [ 572.697864] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 572.697864] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] self.wait() [ 572.697864] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 572.697864] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] self[:] = self._gt.wait() [ 572.697864] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 572.697864] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] return self._exit_event.wait() [ 572.697864] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 572.698274] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] result = hub.switch() [ 572.698274] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 572.698274] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] return self.greenlet.switch() [ 572.698274] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 572.698274] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] result = function(*args, **kwargs) [ 572.698274] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 572.698274] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] return func(*args, **kwargs) [ 572.698274] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 572.698274] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] raise e [ 572.698274] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 572.698274] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] nwinfo = self.network_api.allocate_for_instance( [ 572.698274] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 572.698274] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] created_port_ids = self._update_ports_for_instance( [ 572.698665] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 572.698665] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] with excutils.save_and_reraise_exception(): [ 572.698665] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 572.698665] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] self.force_reraise() [ 572.698665] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 572.698665] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] raise self.value [ 572.698665] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 572.698665] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] updated_port = self._update_port( [ 572.698665] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 572.698665] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] _ensure_no_port_binding_failure(port) [ 572.698665] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 572.698665] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] raise exception.PortBindingFailed(port_id=port['id']) [ 572.698992] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] nova.exception.PortBindingFailed: Binding failed for port 60f230d6-abe8-45a2-896b-081db4709502, please check neutron logs for more information. [ 572.698992] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] [ 572.698992] env[62499]: INFO nova.compute.manager [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Terminating instance [ 572.700485] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Acquiring lock "refresh_cache-76cf7f28-7413-4b06-bd30-ae22bbc2999e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 572.700652] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Acquired lock "refresh_cache-76cf7f28-7413-4b06-bd30-ae22bbc2999e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 572.700825] env[62499]: DEBUG nova.network.neutron [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 572.764808] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 572.815106] env[62499]: DEBUG nova.virt.hardware [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 572.815351] env[62499]: DEBUG nova.virt.hardware [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 572.815505] env[62499]: DEBUG nova.virt.hardware [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 572.817741] env[62499]: DEBUG nova.virt.hardware [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 572.817741] env[62499]: DEBUG nova.virt.hardware [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 572.817741] env[62499]: DEBUG nova.virt.hardware [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 572.817741] env[62499]: DEBUG nova.virt.hardware [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 572.817741] env[62499]: DEBUG nova.virt.hardware [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 572.818131] env[62499]: DEBUG nova.virt.hardware [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 572.818131] env[62499]: DEBUG nova.virt.hardware [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 572.818131] env[62499]: DEBUG nova.virt.hardware [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 572.818131] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f75d5823-b337-4a75-b198-0b8536cb5dd7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 572.828518] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3dcf8e9c-7e24-467d-b75c-365fd0cb1554 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 572.839376] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Instance VIF info [] {{(pid=62499) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 572.844790] env[62499]: DEBUG oslo.service.loopingcall [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 572.848232] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Creating VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 572.848469] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-3e210e52-77c5-4b09-8bad-789e19f8cf03 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 572.868105] env[62499]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 572.868105] env[62499]: value = "task-1335670" [ 572.868105] env[62499]: _type = "Task" [ 572.868105] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 572.876014] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335670, 'name': CreateVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 572.887082] env[62499]: DEBUG nova.network.neutron [req-cc7d185b-dde7-49aa-8358-e66b73a13ad2 req-0d240bbb-93e9-4427-a73a-25fc63222156 service nova] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 573.039526] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 9160ca3a-2aa2-4cfc-9900-3e4c770d2657 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 573.039791] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Total usable vcpus: 48, total allocated vcpus: 6 {{(pid=62499) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 573.039938] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1664MB phys_disk=200GB used_disk=6GB total_vcpus=48 used_vcpus=6 pci_stats=[] {{(pid=62499) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 573.270330] env[62499]: DEBUG nova.network.neutron [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 573.378516] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335670, 'name': CreateVM_Task, 'duration_secs': 0.300359} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 573.378712] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Created VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 573.379601] env[62499]: DEBUG oslo_vmware.service [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e8b3283a-a073-4f5d-b76b-595210ebe398 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.388089] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f116aad6-096f-4657-906e-7de608463cb9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.392641] env[62499]: DEBUG oslo_concurrency.lockutils [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Acquiring lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 573.392641] env[62499]: DEBUG oslo_concurrency.lockutils [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Acquired lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 573.393063] env[62499]: DEBUG oslo_concurrency.lockutils [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Acquired external semaphore "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 573.393309] env[62499]: DEBUG oslo_concurrency.lockutils [req-cc7d185b-dde7-49aa-8358-e66b73a13ad2 req-0d240bbb-93e9-4427-a73a-25fc63222156 service nova] Releasing lock "refresh_cache-2b7122b2-136d-4e0c-97eb-155b33905436" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 573.393627] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-cd326667-5312-4491-927e-9ed6a59dc8cf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.407820] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Waiting for the task: (returnval){ [ 573.407820] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52468e3e-d009-d66c-02e1-491f04efa887" [ 573.407820] env[62499]: _type = "Task" [ 573.407820] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 573.408679] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d52ebc0-c21f-4b1b-9f50-ceccc1729f1f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.448783] env[62499]: DEBUG oslo_concurrency.lockutils [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Releasing lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 573.449163] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Processing image 0422123c-16ce-4c9d-b1c6-0281fe0e54ad {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 573.449416] env[62499]: DEBUG oslo_concurrency.lockutils [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Acquiring lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 573.449566] env[62499]: DEBUG oslo_concurrency.lockutils [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Acquired lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 573.449744] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 573.450285] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-012eb49e-0337-48c4-a939-3ed259b01a4e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.452857] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-745c5d54-c197-4fab-8d1f-896a0d9d0f44 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.465855] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a56b7447-db89-4dfc-bcf7-02fd3f98e3e0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.482261] env[62499]: DEBUG nova.compute.provider_tree [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 573.484746] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Created directory with path [datastore1] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 573.484831] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Folder [datastore1] devstack-image-cache_base created. {{(pid=62499) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 573.486025] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7974b454-7ffb-4dda-a38b-b56b317f1e83 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.492222] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-799a334f-2a1e-4155-b9db-36fdc9b4f95e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.500439] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Waiting for the task: (returnval){ [ 573.500439] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52607fcc-1676-0136-67dc-0354d4d679e4" [ 573.500439] env[62499]: _type = "Task" [ 573.500439] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 573.509323] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52607fcc-1676-0136-67dc-0354d4d679e4, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 573.525105] env[62499]: DEBUG nova.network.neutron [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 573.987662] env[62499]: DEBUG nova.scheduler.client.report [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 574.016759] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Preparing fetch location {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 574.017354] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Creating directory with path [datastore1] vmware_temp/c65a8aa1-beaf-4502-858a-4273c7f93e0a/0422123c-16ce-4c9d-b1c6-0281fe0e54ad {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 574.018037] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-7df94442-e0a5-4b06-85f8-b03b3ca0008f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 574.028400] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Releasing lock "refresh_cache-76cf7f28-7413-4b06-bd30-ae22bbc2999e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 574.028808] env[62499]: DEBUG nova.compute.manager [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 574.029527] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 574.031613] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9ce05a5b-31ba-499f-8ddc-723b8c5486dd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 574.039579] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Created directory with path [datastore1] vmware_temp/c65a8aa1-beaf-4502-858a-4273c7f93e0a/0422123c-16ce-4c9d-b1c6-0281fe0e54ad {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 574.040249] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Fetch image to [datastore1] vmware_temp/c65a8aa1-beaf-4502-858a-4273c7f93e0a/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/tmp-sparse.vmdk {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 574.040341] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Downloading image file data 0422123c-16ce-4c9d-b1c6-0281fe0e54ad to [datastore1] vmware_temp/c65a8aa1-beaf-4502-858a-4273c7f93e0a/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/tmp-sparse.vmdk on the data store datastore1 {{(pid=62499) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 574.043600] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00295ece-9134-4f75-ab81-dc7913b249fb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 574.050282] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd9fb716-60cf-4093-a227-64219b90460a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 574.070217] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2280a484-5e41-4465-9839-6a4c6486d350 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 574.078143] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 76cf7f28-7413-4b06-bd30-ae22bbc2999e could not be found. [ 574.078143] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 574.078297] env[62499]: INFO nova.compute.manager [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Took 0.05 seconds to destroy the instance on the hypervisor. [ 574.078946] env[62499]: DEBUG oslo.service.loopingcall [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 574.079109] env[62499]: DEBUG nova.compute.manager [-] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 574.079176] env[62499]: DEBUG nova.network.neutron [-] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 574.089708] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-497d6822-6a09-4c2d-9bf5-2558aa473259 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 574.128449] env[62499]: DEBUG nova.network.neutron [-] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 574.128449] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d32587ac-b337-486e-a88a-322c5c7c843c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 574.137745] env[62499]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-f5b782c3-6ed5-48ed-9625-5af17d397f2c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 574.174633] env[62499]: DEBUG nova.virt.vmwareapi.images [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Downloading image file data 0422123c-16ce-4c9d-b1c6-0281fe0e54ad to the data store datastore1 {{(pid=62499) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 574.240944] env[62499]: DEBUG oslo_vmware.rw_handles [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/c65a8aa1-beaf-4502-858a-4273c7f93e0a/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=62499) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 574.493195] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62499) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 574.493451] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 7.062s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 574.493710] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 19.335s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 574.631763] env[62499]: DEBUG nova.network.neutron [-] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 575.015552] env[62499]: DEBUG oslo_vmware.rw_handles [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Completed reading data from the image iterator. {{(pid=62499) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 575.015552] env[62499]: DEBUG oslo_vmware.rw_handles [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/c65a8aa1-beaf-4502-858a-4273c7f93e0a/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=62499) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 575.071506] env[62499]: DEBUG nova.virt.vmwareapi.images [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Downloaded image file data 0422123c-16ce-4c9d-b1c6-0281fe0e54ad to vmware_temp/c65a8aa1-beaf-4502-858a-4273c7f93e0a/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/tmp-sparse.vmdk on the data store datastore1 {{(pid=62499) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 575.076718] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Caching image {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 575.077031] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Copying Virtual Disk [datastore1] vmware_temp/c65a8aa1-beaf-4502-858a-4273c7f93e0a/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/tmp-sparse.vmdk to [datastore1] vmware_temp/c65a8aa1-beaf-4502-858a-4273c7f93e0a/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 575.077309] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-6d843e1c-ed61-4155-82b8-89794fc9ae56 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 575.086696] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Waiting for the task: (returnval){ [ 575.086696] env[62499]: value = "task-1335671" [ 575.086696] env[62499]: _type = "Task" [ 575.086696] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 575.095328] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335671, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 575.134904] env[62499]: INFO nova.compute.manager [-] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Took 1.06 seconds to deallocate network for instance. [ 575.141809] env[62499]: DEBUG nova.compute.claims [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 575.141938] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 575.339788] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3385bcbf-fcbb-477f-94a4-23859c854716 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 575.351505] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e6b4d07-ea81-4c4d-9b87-9391e8e8a88a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 575.388499] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5107a27-64a3-4b4a-a681-7fe5a1534714 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 575.397766] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42d9e9f1-e893-4888-846a-7be965b9c00d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 575.412159] env[62499]: DEBUG nova.compute.provider_tree [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 575.595358] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335671, 'name': CopyVirtualDisk_Task} progress is 100%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 575.630181] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Acquiring lock "2a963e2c-7ee3-41a4-842b-06c151d27d84" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 575.630504] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Lock "2a963e2c-7ee3-41a4-842b-06c151d27d84" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 575.843411] env[62499]: DEBUG nova.compute.manager [req-cceba7fe-143e-49a6-993b-930d8a6c6ad0 req-28e12b62-6e02-415f-a8ba-bb26b4c43abc service nova] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Received event network-vif-deleted-42bcc024-8460-4983-9bc4-d3545aa63650 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 575.843736] env[62499]: DEBUG nova.compute.manager [req-cceba7fe-143e-49a6-993b-930d8a6c6ad0 req-28e12b62-6e02-415f-a8ba-bb26b4c43abc service nova] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Received event network-changed-60f230d6-abe8-45a2-896b-081db4709502 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 575.844101] env[62499]: DEBUG nova.compute.manager [req-cceba7fe-143e-49a6-993b-930d8a6c6ad0 req-28e12b62-6e02-415f-a8ba-bb26b4c43abc service nova] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Refreshing instance network info cache due to event network-changed-60f230d6-abe8-45a2-896b-081db4709502. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 575.844101] env[62499]: DEBUG oslo_concurrency.lockutils [req-cceba7fe-143e-49a6-993b-930d8a6c6ad0 req-28e12b62-6e02-415f-a8ba-bb26b4c43abc service nova] Acquiring lock "refresh_cache-76cf7f28-7413-4b06-bd30-ae22bbc2999e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 575.844279] env[62499]: DEBUG oslo_concurrency.lockutils [req-cceba7fe-143e-49a6-993b-930d8a6c6ad0 req-28e12b62-6e02-415f-a8ba-bb26b4c43abc service nova] Acquired lock "refresh_cache-76cf7f28-7413-4b06-bd30-ae22bbc2999e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 575.844645] env[62499]: DEBUG nova.network.neutron [req-cceba7fe-143e-49a6-993b-930d8a6c6ad0 req-28e12b62-6e02-415f-a8ba-bb26b4c43abc service nova] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Refreshing network info cache for port 60f230d6-abe8-45a2-896b-081db4709502 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 575.915746] env[62499]: DEBUG nova.scheduler.client.report [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 576.100778] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335671, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.662635} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 576.101323] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Copied Virtual Disk [datastore1] vmware_temp/c65a8aa1-beaf-4502-858a-4273c7f93e0a/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/tmp-sparse.vmdk to [datastore1] vmware_temp/c65a8aa1-beaf-4502-858a-4273c7f93e0a/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 576.101874] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Deleting the datastore file [datastore1] vmware_temp/c65a8aa1-beaf-4502-858a-4273c7f93e0a/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/tmp-sparse.vmdk {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 576.102298] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-6c05c048-6467-4265-bcb0-a6ce1b74cebd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 576.111944] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Waiting for the task: (returnval){ [ 576.111944] env[62499]: value = "task-1335672" [ 576.111944] env[62499]: _type = "Task" [ 576.111944] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 576.124438] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335672, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 576.374045] env[62499]: DEBUG nova.network.neutron [req-cceba7fe-143e-49a6-993b-930d8a6c6ad0 req-28e12b62-6e02-415f-a8ba-bb26b4c43abc service nova] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 576.421859] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.927s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 576.421996] env[62499]: ERROR nova.compute.manager [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port eff92d51-a578-421d-98e9-d5ffbfb11bb1, please check neutron logs for more information. [ 576.421996] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Traceback (most recent call last): [ 576.421996] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 576.421996] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] self.driver.spawn(context, instance, image_meta, [ 576.421996] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 576.421996] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 576.421996] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 576.421996] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] vm_ref = self.build_virtual_machine(instance, [ 576.421996] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 576.421996] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] vif_infos = vmwarevif.get_vif_info(self._session, [ 576.421996] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 576.423455] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] for vif in network_info: [ 576.423455] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 576.423455] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] return self._sync_wrapper(fn, *args, **kwargs) [ 576.423455] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 576.423455] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] self.wait() [ 576.423455] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 576.423455] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] self[:] = self._gt.wait() [ 576.423455] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 576.423455] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] return self._exit_event.wait() [ 576.423455] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 576.423455] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] result = hub.switch() [ 576.423455] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 576.423455] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] return self.greenlet.switch() [ 576.423865] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 576.423865] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] result = function(*args, **kwargs) [ 576.423865] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 576.423865] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] return func(*args, **kwargs) [ 576.423865] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 576.423865] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] raise e [ 576.423865] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 576.423865] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] nwinfo = self.network_api.allocate_for_instance( [ 576.423865] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 576.423865] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] created_port_ids = self._update_ports_for_instance( [ 576.423865] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 576.423865] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] with excutils.save_and_reraise_exception(): [ 576.423865] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 576.424390] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] self.force_reraise() [ 576.424390] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 576.424390] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] raise self.value [ 576.424390] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 576.424390] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] updated_port = self._update_port( [ 576.424390] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 576.424390] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] _ensure_no_port_binding_failure(port) [ 576.424390] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 576.424390] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] raise exception.PortBindingFailed(port_id=port['id']) [ 576.424390] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] nova.exception.PortBindingFailed: Binding failed for port eff92d51-a578-421d-98e9-d5ffbfb11bb1, please check neutron logs for more information. [ 576.424390] env[62499]: ERROR nova.compute.manager [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] [ 576.424706] env[62499]: DEBUG nova.compute.utils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Binding failed for port eff92d51-a578-421d-98e9-d5ffbfb11bb1, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 576.424706] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 20.576s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 576.427139] env[62499]: INFO nova.compute.claims [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 576.429702] env[62499]: DEBUG nova.compute.manager [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Build of instance 48fc0cbf-18a6-4bbc-8455-93800b911d6a was re-scheduled: Binding failed for port eff92d51-a578-421d-98e9-d5ffbfb11bb1, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 576.430215] env[62499]: DEBUG nova.compute.manager [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 576.430403] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Acquiring lock "refresh_cache-48fc0cbf-18a6-4bbc-8455-93800b911d6a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 576.430572] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Acquired lock "refresh_cache-48fc0cbf-18a6-4bbc-8455-93800b911d6a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 576.430697] env[62499]: DEBUG nova.network.neutron [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 576.465158] env[62499]: DEBUG nova.network.neutron [req-cceba7fe-143e-49a6-993b-930d8a6c6ad0 req-28e12b62-6e02-415f-a8ba-bb26b4c43abc service nova] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 576.622632] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335672, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.021988} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 576.622909] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Deleted the datastore file {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 576.623141] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Moving file from [datastore1] vmware_temp/c65a8aa1-beaf-4502-858a-4273c7f93e0a/0422123c-16ce-4c9d-b1c6-0281fe0e54ad to [datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad. {{(pid=62499) file_move /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:334}} [ 576.623396] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.MoveDatastoreFile_Task with opID=oslo.vmware-921ff5d4-e254-49ab-9e9e-a69d30f87913 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 576.632411] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Waiting for the task: (returnval){ [ 576.632411] env[62499]: value = "task-1335673" [ 576.632411] env[62499]: _type = "Task" [ 576.632411] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 576.640994] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335673, 'name': MoveDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 576.963068] env[62499]: DEBUG nova.network.neutron [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 576.968236] env[62499]: DEBUG oslo_concurrency.lockutils [req-cceba7fe-143e-49a6-993b-930d8a6c6ad0 req-28e12b62-6e02-415f-a8ba-bb26b4c43abc service nova] Releasing lock "refresh_cache-76cf7f28-7413-4b06-bd30-ae22bbc2999e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 576.968236] env[62499]: DEBUG nova.compute.manager [req-cceba7fe-143e-49a6-993b-930d8a6c6ad0 req-28e12b62-6e02-415f-a8ba-bb26b4c43abc service nova] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Received event network-vif-deleted-60f230d6-abe8-45a2-896b-081db4709502 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 577.039748] env[62499]: DEBUG nova.network.neutron [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 577.144487] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335673, 'name': MoveDatastoreFile_Task, 'duration_secs': 0.025466} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 577.145245] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] File moved {{(pid=62499) file_move /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:346}} [ 577.145641] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Cleaning up location [datastore1] vmware_temp/c65a8aa1-beaf-4502-858a-4273c7f93e0a {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:640}} [ 577.146018] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Deleting the datastore file [datastore1] vmware_temp/c65a8aa1-beaf-4502-858a-4273c7f93e0a {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 577.146403] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-36346f39-ddba-4ddb-958f-9fe7cb6de0f5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 577.154530] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Waiting for the task: (returnval){ [ 577.154530] env[62499]: value = "task-1335674" [ 577.154530] env[62499]: _type = "Task" [ 577.154530] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 577.162672] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335674, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 577.543921] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Releasing lock "refresh_cache-48fc0cbf-18a6-4bbc-8455-93800b911d6a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 577.543921] env[62499]: DEBUG nova.compute.manager [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 577.543921] env[62499]: DEBUG nova.compute.manager [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 577.543921] env[62499]: DEBUG nova.network.neutron [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 577.593408] env[62499]: DEBUG nova.network.neutron [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 577.670873] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335674, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.028446} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 577.671365] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Deleted the datastore file {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 577.672119] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-404f70c8-4d4d-4398-a8fe-0126cf25795e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 577.677771] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Waiting for the task: (returnval){ [ 577.677771] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52b13004-2e5e-6df8-eef6-a9f7ead7c818" [ 577.677771] env[62499]: _type = "Task" [ 577.677771] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 577.690774] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52b13004-2e5e-6df8-eef6-a9f7ead7c818, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 577.837020] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0611e9c6-f56f-4c03-8841-d9f7b4889dc3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 577.847335] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a431e0df-776d-414d-9b51-f1c79a46e240 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 577.884586] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6026c6e1-a906-4c7a-924f-bece91dfc2bc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 577.893305] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c0fccbc-5413-49c8-8a06-04e4b4db2336 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 577.910873] env[62499]: DEBUG nova.compute.provider_tree [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 578.098316] env[62499]: DEBUG nova.network.neutron [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 578.142885] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Acquiring lock "5e18d3cc-62bf-4f12-8e89-2e1bcad15d49" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 578.142885] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Lock "5e18d3cc-62bf-4f12-8e89-2e1bcad15d49" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 578.189408] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52b13004-2e5e-6df8-eef6-a9f7ead7c818, 'name': SearchDatastore_Task, 'duration_secs': 0.01091} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 578.189703] env[62499]: DEBUG oslo_concurrency.lockutils [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Releasing lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 578.189920] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Copying Virtual Disk [datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore1] ddd212aa-9f4a-4b9d-aa60-cd4b64580f84/ddd212aa-9f4a-4b9d-aa60-cd4b64580f84.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 578.190313] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-56cb35bf-8181-4a22-9389-aec91c0e6d13 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 578.197338] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Waiting for the task: (returnval){ [ 578.197338] env[62499]: value = "task-1335675" [ 578.197338] env[62499]: _type = "Task" [ 578.197338] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 578.207456] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335675, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 578.415848] env[62499]: DEBUG nova.scheduler.client.report [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 578.602277] env[62499]: INFO nova.compute.manager [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: 48fc0cbf-18a6-4bbc-8455-93800b911d6a] Took 1.06 seconds to deallocate network for instance. [ 578.710095] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335675, 'name': CopyVirtualDisk_Task} progress is 77%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 578.922345] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.497s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 578.922345] env[62499]: DEBUG nova.compute.manager [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 578.927351] env[62499]: DEBUG oslo_concurrency.lockutils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 22.522s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 579.212219] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335675, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.528815} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 579.212219] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Copied Virtual Disk [datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore1] ddd212aa-9f4a-4b9d-aa60-cd4b64580f84/ddd212aa-9f4a-4b9d-aa60-cd4b64580f84.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 579.212521] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Extending root virtual disk to 1048576 {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 579.213518] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-7dfef161-6d25-4e16-b4fd-7e371cce08e3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 579.221987] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Waiting for the task: (returnval){ [ 579.221987] env[62499]: value = "task-1335680" [ 579.221987] env[62499]: _type = "Task" [ 579.221987] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 579.233999] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335680, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 579.434247] env[62499]: DEBUG nova.compute.utils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 579.444178] env[62499]: DEBUG nova.compute.manager [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 579.444178] env[62499]: DEBUG nova.network.neutron [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 579.660462] env[62499]: INFO nova.scheduler.client.report [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Deleted allocations for instance 48fc0cbf-18a6-4bbc-8455-93800b911d6a [ 579.734054] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335680, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.070581} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 579.740928] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Extended root virtual disk {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 579.742390] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1125b914-8dfa-4835-9f23-573cb52752ae {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 579.769311] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Reconfiguring VM instance instance-00000004 to attach disk [datastore1] ddd212aa-9f4a-4b9d-aa60-cd4b64580f84/ddd212aa-9f4a-4b9d-aa60-cd4b64580f84.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 579.777050] env[62499]: DEBUG nova.policy [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b50bcb588c294e339ed3bc90b34ec2cc', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ec3b357a819b4d7cbfd5fb7cfd6110c3', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 579.778898] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-dc76ccec-b598-496f-89cc-9ffcf55d5cb1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 579.802884] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Waiting for the task: (returnval){ [ 579.802884] env[62499]: value = "task-1335681" [ 579.802884] env[62499]: _type = "Task" [ 579.802884] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 579.815344] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335681, 'name': ReconfigVM_Task} progress is 10%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 579.903279] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a5f2c0c8-3994-4082-9eb2-b8ee2f416e81 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 579.911751] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc8597e5-9a92-409c-ba99-3851f3e7b552 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 579.945059] env[62499]: DEBUG nova.compute.manager [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 579.955869] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52a3718c-361a-4c0b-8171-ad66578f3d3f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 579.964147] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bbbcf9c4-b7f0-4525-bf70-5c09c2259b50 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 579.979938] env[62499]: DEBUG nova.compute.provider_tree [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 580.170680] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01135f88-abb2-4a89-84bb-7d712af48c5b tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Lock "48fc0cbf-18a6-4bbc-8455-93800b911d6a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 43.485s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 580.316491] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335681, 'name': ReconfigVM_Task} progress is 99%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 580.485747] env[62499]: DEBUG nova.scheduler.client.report [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 580.649043] env[62499]: DEBUG nova.network.neutron [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Successfully created port: 149941b7-e36f-4036-b7de-c89202397eac {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 580.672938] env[62499]: DEBUG nova.compute.manager [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 580.817319] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335681, 'name': ReconfigVM_Task} progress is 99%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 580.970957] env[62499]: DEBUG nova.compute.manager [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 580.995921] env[62499]: DEBUG oslo_concurrency.lockutils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.069s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 580.996208] env[62499]: ERROR nova.compute.manager [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f71924e4-d6df-4e41-b251-783bd1c767a7, please check neutron logs for more information. [ 580.996208] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Traceback (most recent call last): [ 580.996208] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 580.996208] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] self.driver.spawn(context, instance, image_meta, [ 580.996208] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 580.996208] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 580.996208] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 580.996208] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] vm_ref = self.build_virtual_machine(instance, [ 580.996208] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 580.996208] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] vif_infos = vmwarevif.get_vif_info(self._session, [ 580.996208] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 580.996632] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] for vif in network_info: [ 580.996632] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 580.996632] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] return self._sync_wrapper(fn, *args, **kwargs) [ 580.996632] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 580.996632] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] self.wait() [ 580.996632] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 580.996632] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] self[:] = self._gt.wait() [ 580.996632] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 580.996632] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] return self._exit_event.wait() [ 580.996632] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 580.996632] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] result = hub.switch() [ 580.996632] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 580.996632] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] return self.greenlet.switch() [ 580.996997] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 580.996997] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] result = function(*args, **kwargs) [ 580.996997] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 580.996997] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] return func(*args, **kwargs) [ 580.996997] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 580.996997] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] raise e [ 580.996997] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 580.996997] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] nwinfo = self.network_api.allocate_for_instance( [ 580.996997] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 580.996997] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] created_port_ids = self._update_ports_for_instance( [ 580.996997] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 580.996997] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] with excutils.save_and_reraise_exception(): [ 580.996997] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 580.999350] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] self.force_reraise() [ 580.999350] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 580.999350] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] raise self.value [ 580.999350] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 580.999350] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] updated_port = self._update_port( [ 580.999350] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 580.999350] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] _ensure_no_port_binding_failure(port) [ 580.999350] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 580.999350] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] raise exception.PortBindingFailed(port_id=port['id']) [ 580.999350] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] nova.exception.PortBindingFailed: Binding failed for port f71924e4-d6df-4e41-b251-783bd1c767a7, please check neutron logs for more information. [ 580.999350] env[62499]: ERROR nova.compute.manager [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] [ 580.999690] env[62499]: DEBUG nova.compute.utils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Binding failed for port f71924e4-d6df-4e41-b251-783bd1c767a7, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 580.999690] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 23.455s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 581.001113] env[62499]: INFO nova.compute.claims [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 581.011326] env[62499]: DEBUG nova.virt.hardware [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 581.011586] env[62499]: DEBUG nova.virt.hardware [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 581.011737] env[62499]: DEBUG nova.virt.hardware [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 581.014343] env[62499]: DEBUG nova.virt.hardware [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 581.016986] env[62499]: DEBUG nova.virt.hardware [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 581.016986] env[62499]: DEBUG nova.virt.hardware [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 581.016986] env[62499]: DEBUG nova.virt.hardware [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 581.016986] env[62499]: DEBUG nova.virt.hardware [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 581.016986] env[62499]: DEBUG nova.virt.hardware [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 581.017336] env[62499]: DEBUG nova.virt.hardware [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 581.017336] env[62499]: DEBUG nova.virt.hardware [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 581.017336] env[62499]: DEBUG nova.compute.manager [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Build of instance 99f77fa6-a625-4771-83c9-012f75ed1c7d was re-scheduled: Binding failed for port f71924e4-d6df-4e41-b251-783bd1c767a7, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 581.017336] env[62499]: DEBUG nova.compute.manager [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 581.017336] env[62499]: DEBUG oslo_concurrency.lockutils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Acquiring lock "refresh_cache-99f77fa6-a625-4771-83c9-012f75ed1c7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 581.017633] env[62499]: DEBUG oslo_concurrency.lockutils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Acquired lock "refresh_cache-99f77fa6-a625-4771-83c9-012f75ed1c7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 581.017862] env[62499]: DEBUG nova.network.neutron [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 581.025981] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d21096a4-86e7-474a-873a-d26a6c98ba28 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 581.042289] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dc3d2a20-fc9f-4364-a9e4-19d673f27ad9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 581.208395] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 581.319311] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335681, 'name': ReconfigVM_Task, 'duration_secs': 1.452039} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 581.319311] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Reconfigured VM instance instance-00000004 to attach disk [datastore1] ddd212aa-9f4a-4b9d-aa60-cd4b64580f84/ddd212aa-9f4a-4b9d-aa60-cd4b64580f84.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 581.320162] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-dd27f828-3a0b-4fba-8693-cf5e66c64346 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 581.326041] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Waiting for the task: (returnval){ [ 581.326041] env[62499]: value = "task-1335683" [ 581.326041] env[62499]: _type = "Task" [ 581.326041] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 581.335147] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335683, 'name': Rename_Task} progress is 5%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 581.569511] env[62499]: DEBUG nova.network.neutron [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 581.711282] env[62499]: DEBUG nova.network.neutron [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 581.840973] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335683, 'name': Rename_Task, 'duration_secs': 0.154257} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 581.841616] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Powering on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 581.842034] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-dad64a4c-8006-4aef-b744-3a37f6185ef7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 581.848297] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Waiting for the task: (returnval){ [ 581.848297] env[62499]: value = "task-1335684" [ 581.848297] env[62499]: _type = "Task" [ 581.848297] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 581.856881] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335684, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 582.217557] env[62499]: DEBUG oslo_concurrency.lockutils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Releasing lock "refresh_cache-99f77fa6-a625-4771-83c9-012f75ed1c7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 582.218387] env[62499]: DEBUG nova.compute.manager [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 582.219827] env[62499]: DEBUG nova.compute.manager [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 582.220331] env[62499]: DEBUG nova.network.neutron [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 582.261750] env[62499]: DEBUG nova.network.neutron [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 582.367311] env[62499]: DEBUG oslo_vmware.api [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Task: {'id': task-1335684, 'name': PowerOnVM_Task, 'duration_secs': 0.476193} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 582.367311] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Powered on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 582.367311] env[62499]: DEBUG nova.compute.manager [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 582.367311] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4baf09e2-96b0-4565-825a-1853e8a01e05 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.387027] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd45239f-5a6d-47c9-9f2a-6ce2f72eb903 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.393931] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f4937596-076d-447c-a01a-c7d2fb111b9d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.431541] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8cfebb63-c872-40ba-978d-1c4c01aa70cc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.439440] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c52101e1-2fc6-4659-a10b-ec767203540f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.454867] env[62499]: DEBUG nova.compute.provider_tree [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 582.490156] env[62499]: DEBUG nova.compute.manager [None req-4117366f-813e-4124-a6dc-627cb55910d8 tempest-ServerDiagnosticsV248Test-1831854974 tempest-ServerDiagnosticsV248Test-1831854974-project-admin] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 582.491557] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a610f16-e95c-49cf-855f-e8ca1e9e4fe5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.498252] env[62499]: INFO nova.compute.manager [None req-4117366f-813e-4124-a6dc-627cb55910d8 tempest-ServerDiagnosticsV248Test-1831854974 tempest-ServerDiagnosticsV248Test-1831854974-project-admin] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Retrieving diagnostics [ 582.499407] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec107d67-7ee2-4bd8-b585-ddfa138332a4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.731473] env[62499]: ERROR nova.compute.manager [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 149941b7-e36f-4036-b7de-c89202397eac, please check neutron logs for more information. [ 582.731473] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 582.731473] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 582.731473] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 582.731473] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 582.731473] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 582.731473] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 582.731473] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 582.731473] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 582.731473] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 582.731473] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 582.731473] env[62499]: ERROR nova.compute.manager raise self.value [ 582.731473] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 582.731473] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 582.731473] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 582.731473] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 582.732237] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 582.732237] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 582.732237] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 149941b7-e36f-4036-b7de-c89202397eac, please check neutron logs for more information. [ 582.732237] env[62499]: ERROR nova.compute.manager [ 582.732237] env[62499]: Traceback (most recent call last): [ 582.732237] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 582.732237] env[62499]: listener.cb(fileno) [ 582.732237] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 582.732237] env[62499]: result = function(*args, **kwargs) [ 582.732237] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 582.732237] env[62499]: return func(*args, **kwargs) [ 582.732237] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 582.732237] env[62499]: raise e [ 582.732237] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 582.732237] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 582.732237] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 582.732237] env[62499]: created_port_ids = self._update_ports_for_instance( [ 582.732237] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 582.732237] env[62499]: with excutils.save_and_reraise_exception(): [ 582.732237] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 582.732237] env[62499]: self.force_reraise() [ 582.732237] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 582.732237] env[62499]: raise self.value [ 582.732237] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 582.732237] env[62499]: updated_port = self._update_port( [ 582.732237] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 582.732237] env[62499]: _ensure_no_port_binding_failure(port) [ 582.732237] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 582.732237] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 582.733482] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 149941b7-e36f-4036-b7de-c89202397eac, please check neutron logs for more information. [ 582.733482] env[62499]: Removing descriptor: 16 [ 582.734277] env[62499]: ERROR nova.compute.manager [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 149941b7-e36f-4036-b7de-c89202397eac, please check neutron logs for more information. [ 582.734277] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Traceback (most recent call last): [ 582.734277] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 582.734277] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] yield resources [ 582.734277] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 582.734277] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] self.driver.spawn(context, instance, image_meta, [ 582.734277] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 582.734277] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 582.734277] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 582.734277] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] vm_ref = self.build_virtual_machine(instance, [ 582.734277] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 582.734651] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] vif_infos = vmwarevif.get_vif_info(self._session, [ 582.734651] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 582.734651] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] for vif in network_info: [ 582.734651] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 582.734651] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] return self._sync_wrapper(fn, *args, **kwargs) [ 582.734651] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 582.734651] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] self.wait() [ 582.734651] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 582.734651] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] self[:] = self._gt.wait() [ 582.734651] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 582.734651] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] return self._exit_event.wait() [ 582.734651] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 582.734651] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] result = hub.switch() [ 582.735048] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 582.735048] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] return self.greenlet.switch() [ 582.735048] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 582.735048] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] result = function(*args, **kwargs) [ 582.735048] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 582.735048] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] return func(*args, **kwargs) [ 582.735048] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 582.735048] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] raise e [ 582.735048] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 582.735048] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] nwinfo = self.network_api.allocate_for_instance( [ 582.735048] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 582.735048] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] created_port_ids = self._update_ports_for_instance( [ 582.735048] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 582.735772] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] with excutils.save_and_reraise_exception(): [ 582.735772] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 582.735772] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] self.force_reraise() [ 582.735772] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 582.735772] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] raise self.value [ 582.735772] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 582.735772] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] updated_port = self._update_port( [ 582.735772] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 582.735772] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] _ensure_no_port_binding_failure(port) [ 582.735772] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 582.735772] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] raise exception.PortBindingFailed(port_id=port['id']) [ 582.735772] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] nova.exception.PortBindingFailed: Binding failed for port 149941b7-e36f-4036-b7de-c89202397eac, please check neutron logs for more information. [ 582.735772] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] [ 582.736552] env[62499]: INFO nova.compute.manager [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Terminating instance [ 582.737693] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Acquiring lock "refresh_cache-8d45497d-ba77-45b1-91f3-719b2689bee3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 582.737934] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Acquired lock "refresh_cache-8d45497d-ba77-45b1-91f3-719b2689bee3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 582.738150] env[62499]: DEBUG nova.network.neutron [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 582.764630] env[62499]: DEBUG nova.network.neutron [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 582.890234] env[62499]: DEBUG oslo_concurrency.lockutils [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 582.957852] env[62499]: DEBUG nova.scheduler.client.report [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 583.268804] env[62499]: DEBUG nova.network.neutron [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 583.272239] env[62499]: INFO nova.compute.manager [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 99f77fa6-a625-4771-83c9-012f75ed1c7d] Took 1.05 seconds to deallocate network for instance. [ 583.466482] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.468s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 583.469795] env[62499]: DEBUG nova.compute.manager [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 583.473306] env[62499]: DEBUG oslo_concurrency.lockutils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 24.100s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 583.475172] env[62499]: INFO nova.compute.claims [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 583.585068] env[62499]: DEBUG oslo_concurrency.lockutils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Acquiring lock "0464da14-cb61-4f8f-9bd7-16f970aec0e7" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 583.586125] env[62499]: DEBUG oslo_concurrency.lockutils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Lock "0464da14-cb61-4f8f-9bd7-16f970aec0e7" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 583.656596] env[62499]: DEBUG nova.network.neutron [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 583.666516] env[62499]: DEBUG nova.compute.manager [req-ff1b3a2e-b0f5-41a1-8909-eb3e86260637 req-04242d69-943b-4f61-a6cb-da9ba33130ac service nova] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Received event network-changed-149941b7-e36f-4036-b7de-c89202397eac {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 583.666645] env[62499]: DEBUG nova.compute.manager [req-ff1b3a2e-b0f5-41a1-8909-eb3e86260637 req-04242d69-943b-4f61-a6cb-da9ba33130ac service nova] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Refreshing instance network info cache due to event network-changed-149941b7-e36f-4036-b7de-c89202397eac. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 583.666799] env[62499]: DEBUG oslo_concurrency.lockutils [req-ff1b3a2e-b0f5-41a1-8909-eb3e86260637 req-04242d69-943b-4f61-a6cb-da9ba33130ac service nova] Acquiring lock "refresh_cache-8d45497d-ba77-45b1-91f3-719b2689bee3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 583.983031] env[62499]: DEBUG nova.compute.utils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 583.988197] env[62499]: DEBUG nova.compute.manager [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 583.988735] env[62499]: DEBUG nova.network.neutron [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 584.099188] env[62499]: DEBUG nova.policy [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd960e74fead14422b11cd3a2ee5df044', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5dbcb6d3c50043ebb6038807c1fe5c7e', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 584.159557] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Releasing lock "refresh_cache-8d45497d-ba77-45b1-91f3-719b2689bee3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 584.161394] env[62499]: DEBUG nova.compute.manager [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 584.161704] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 584.162080] env[62499]: DEBUG oslo_concurrency.lockutils [req-ff1b3a2e-b0f5-41a1-8909-eb3e86260637 req-04242d69-943b-4f61-a6cb-da9ba33130ac service nova] Acquired lock "refresh_cache-8d45497d-ba77-45b1-91f3-719b2689bee3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 584.162284] env[62499]: DEBUG nova.network.neutron [req-ff1b3a2e-b0f5-41a1-8909-eb3e86260637 req-04242d69-943b-4f61-a6cb-da9ba33130ac service nova] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Refreshing network info cache for port 149941b7-e36f-4036-b7de-c89202397eac {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 584.163526] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ad389255-08c8-49c6-8ba4-1052b13ceff0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.174752] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3c6c7840-4345-4d30-9ae1-73d1418a56cb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.206555] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 8d45497d-ba77-45b1-91f3-719b2689bee3 could not be found. [ 584.206851] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 584.207054] env[62499]: INFO nova.compute.manager [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Took 0.05 seconds to destroy the instance on the hypervisor. [ 584.207456] env[62499]: DEBUG oslo.service.loopingcall [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 584.208190] env[62499]: DEBUG nova.compute.manager [-] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 584.208297] env[62499]: DEBUG nova.network.neutron [-] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 584.237334] env[62499]: DEBUG nova.network.neutron [-] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 584.324093] env[62499]: INFO nova.scheduler.client.report [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Deleted allocations for instance 99f77fa6-a625-4771-83c9-012f75ed1c7d [ 584.492906] env[62499]: DEBUG nova.compute.manager [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 584.512605] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Acquiring lock "27103ce2-0962-4fce-9331-d74179b0510c" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 584.513662] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Lock "27103ce2-0962-4fce-9331-d74179b0510c" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 584.513662] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Acquiring lock "27103ce2-0962-4fce-9331-d74179b0510c-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 584.513662] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Lock "27103ce2-0962-4fce-9331-d74179b0510c-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 584.513662] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Lock "27103ce2-0962-4fce-9331-d74179b0510c-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 584.524272] env[62499]: INFO nova.compute.manager [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Terminating instance [ 584.526372] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Acquiring lock "refresh_cache-27103ce2-0962-4fce-9331-d74179b0510c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 584.526531] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Acquired lock "refresh_cache-27103ce2-0962-4fce-9331-d74179b0510c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 584.527530] env[62499]: DEBUG nova.network.neutron [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 584.716448] env[62499]: DEBUG nova.network.neutron [req-ff1b3a2e-b0f5-41a1-8909-eb3e86260637 req-04242d69-943b-4f61-a6cb-da9ba33130ac service nova] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 584.741155] env[62499]: DEBUG nova.network.neutron [-] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 584.830914] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f02d11f4-4f52-4b29-9c85-0ec646eb5cd3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.835868] env[62499]: DEBUG oslo_concurrency.lockutils [None req-674d8a4c-11a0-46bb-90a5-40d4f14ed82c tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Lock "99f77fa6-a625-4771-83c9-012f75ed1c7d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 45.997s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 584.841599] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3c25fe70-f5f4-47b6-b942-bb27037c31d7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.875578] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-283db30f-e82d-45c5-9537-efabe098e81d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.887360] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4707a31a-083c-4679-a298-791028b20493 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.902170] env[62499]: DEBUG nova.compute.provider_tree [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 585.021425] env[62499]: DEBUG nova.network.neutron [req-ff1b3a2e-b0f5-41a1-8909-eb3e86260637 req-04242d69-943b-4f61-a6cb-da9ba33130ac service nova] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 585.083436] env[62499]: DEBUG nova.network.neutron [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 585.098973] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquiring lock "ddd212aa-9f4a-4b9d-aa60-cd4b64580f84" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 585.099379] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Lock "ddd212aa-9f4a-4b9d-aa60-cd4b64580f84" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 585.099546] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquiring lock "ddd212aa-9f4a-4b9d-aa60-cd4b64580f84-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 585.099612] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Lock "ddd212aa-9f4a-4b9d-aa60-cd4b64580f84-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 585.101437] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Lock "ddd212aa-9f4a-4b9d-aa60-cd4b64580f84-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 585.101871] env[62499]: INFO nova.compute.manager [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Terminating instance [ 585.106414] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquiring lock "refresh_cache-ddd212aa-9f4a-4b9d-aa60-cd4b64580f84" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 585.106572] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquired lock "refresh_cache-ddd212aa-9f4a-4b9d-aa60-cd4b64580f84" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 585.106744] env[62499]: DEBUG nova.network.neutron [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 585.134467] env[62499]: DEBUG nova.network.neutron [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Successfully created port: 1bf2c3a4-bfd7-4a69-a486-3f07a007056e {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 585.245151] env[62499]: INFO nova.compute.manager [-] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Took 1.04 seconds to deallocate network for instance. [ 585.247995] env[62499]: DEBUG nova.compute.claims [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 585.247995] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 585.283539] env[62499]: DEBUG nova.network.neutron [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 585.340340] env[62499]: DEBUG nova.compute.manager [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 585.405028] env[62499]: DEBUG nova.scheduler.client.report [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 585.506495] env[62499]: DEBUG nova.compute.manager [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 585.527515] env[62499]: DEBUG oslo_concurrency.lockutils [req-ff1b3a2e-b0f5-41a1-8909-eb3e86260637 req-04242d69-943b-4f61-a6cb-da9ba33130ac service nova] Releasing lock "refresh_cache-8d45497d-ba77-45b1-91f3-719b2689bee3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 585.542105] env[62499]: DEBUG nova.virt.hardware [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:33:25Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='2139270534',id=25,is_public=True,memory_mb=192,name='tempest-flavor_with_ephemeral_0-555035406',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 585.542363] env[62499]: DEBUG nova.virt.hardware [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 585.542363] env[62499]: DEBUG nova.virt.hardware [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 585.542435] env[62499]: DEBUG nova.virt.hardware [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 585.542580] env[62499]: DEBUG nova.virt.hardware [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 585.542732] env[62499]: DEBUG nova.virt.hardware [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 585.543467] env[62499]: DEBUG nova.virt.hardware [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 585.543849] env[62499]: DEBUG nova.virt.hardware [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 585.544095] env[62499]: DEBUG nova.virt.hardware [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 585.544309] env[62499]: DEBUG nova.virt.hardware [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 585.544627] env[62499]: DEBUG nova.virt.hardware [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 585.545505] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cac1b61d-48b6-47c9-abff-00f69eb7ccf2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 585.554880] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-816de1a6-5a0d-4019-a1af-7f747877adfe {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 585.781269] env[62499]: DEBUG nova.network.neutron [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 585.791857] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Releasing lock "refresh_cache-27103ce2-0962-4fce-9331-d74179b0510c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 585.791857] env[62499]: DEBUG nova.compute.manager [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 585.791857] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 585.791857] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4916da1a-67ff-4125-9ea3-28ae90ee60c6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 585.800989] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Powering off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 585.800989] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-0678e103-eab9-4618-b6f2-4bd959830e74 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 585.809306] env[62499]: DEBUG oslo_vmware.api [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Waiting for the task: (returnval){ [ 585.809306] env[62499]: value = "task-1335687" [ 585.809306] env[62499]: _type = "Task" [ 585.809306] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 585.822551] env[62499]: DEBUG oslo_vmware.api [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': task-1335687, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 585.876795] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 585.912052] env[62499]: DEBUG oslo_concurrency.lockutils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.438s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 585.912052] env[62499]: DEBUG nova.compute.manager [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 585.915204] env[62499]: DEBUG oslo_concurrency.lockutils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 24.601s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 585.916662] env[62499]: INFO nova.compute.claims [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 585.940593] env[62499]: DEBUG nova.network.neutron [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 586.322878] env[62499]: DEBUG oslo_vmware.api [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': task-1335687, 'name': PowerOffVM_Task, 'duration_secs': 0.131413} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 586.323037] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Powered off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 586.323258] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Unregistering the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 586.323524] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-bb0e2ba7-0540-49ac-9688-847c0a2e2cd0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 586.348655] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Unregistered the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 586.348655] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Deleting contents of the VM from datastore datastore2 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 586.348655] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Deleting the datastore file [datastore2] 27103ce2-0962-4fce-9331-d74179b0510c {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 586.348655] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-cbc954f3-b73d-4681-8722-607ae7623ffc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 586.356295] env[62499]: DEBUG oslo_vmware.api [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Waiting for the task: (returnval){ [ 586.356295] env[62499]: value = "task-1335689" [ 586.356295] env[62499]: _type = "Task" [ 586.356295] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 586.365546] env[62499]: DEBUG oslo_vmware.api [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': task-1335689, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 586.424171] env[62499]: DEBUG nova.compute.utils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 586.425291] env[62499]: DEBUG nova.compute.manager [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 586.425710] env[62499]: DEBUG nova.network.neutron [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 586.442857] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Releasing lock "refresh_cache-ddd212aa-9f4a-4b9d-aa60-cd4b64580f84" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 586.443424] env[62499]: DEBUG nova.compute.manager [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 586.443737] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 586.444898] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8725b67e-f4bf-4388-9260-9dac43c26726 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 586.454012] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Powering off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 586.454459] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-2fe144cf-d26c-4379-a405-199968fe36a5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 586.461705] env[62499]: DEBUG oslo_vmware.api [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 586.461705] env[62499]: value = "task-1335690" [ 586.461705] env[62499]: _type = "Task" [ 586.461705] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 586.469524] env[62499]: DEBUG oslo_vmware.api [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335690, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 586.512883] env[62499]: DEBUG nova.compute.manager [req-3c2a0aac-6516-4227-9075-a73521e41316 req-f3e3374c-78e1-4fce-a9ea-aace668fb261 service nova] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Received event network-vif-deleted-149941b7-e36f-4036-b7de-c89202397eac {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 586.515963] env[62499]: DEBUG nova.policy [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e8ac2ea6b5b2415e97542da5647a644a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '05837bc39d0040dbbcff7fe1236b5221', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 586.868915] env[62499]: DEBUG oslo_vmware.api [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Task: {'id': task-1335689, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.200067} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 586.868915] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Deleted the datastore file {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 586.870640] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Deleted contents of the VM from datastore datastore2 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 586.870640] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 586.870640] env[62499]: INFO nova.compute.manager [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Took 1.08 seconds to destroy the instance on the hypervisor. [ 586.870640] env[62499]: DEBUG oslo.service.loopingcall [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 586.870640] env[62499]: DEBUG nova.compute.manager [-] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 586.871312] env[62499]: DEBUG nova.network.neutron [-] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 586.931778] env[62499]: DEBUG nova.compute.manager [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 586.941081] env[62499]: DEBUG nova.network.neutron [-] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 586.977564] env[62499]: DEBUG oslo_vmware.api [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335690, 'name': PowerOffVM_Task, 'duration_secs': 0.133888} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 586.977991] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Powered off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 586.978101] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Unregistering the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 586.978522] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-f1f17f8a-ecc2-46c8-9613-921f0f024ad1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 587.007868] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Unregistered the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 587.007868] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Deleting contents of the VM from datastore datastore1 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 587.008023] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Deleting the datastore file [datastore1] ddd212aa-9f4a-4b9d-aa60-cd4b64580f84 {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 587.012154] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-750e4edb-04d5-47d2-a3d5-51c1b2c164e6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 587.021089] env[62499]: DEBUG oslo_vmware.api [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for the task: (returnval){ [ 587.021089] env[62499]: value = "task-1335692" [ 587.021089] env[62499]: _type = "Task" [ 587.021089] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 587.032832] env[62499]: DEBUG oslo_vmware.api [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335692, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 587.160040] env[62499]: DEBUG nova.network.neutron [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Successfully created port: 3ecff7db-5412-4e0b-bc53-9a1924aa0f90 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 587.280901] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e127e2e1-7ec6-4c96-b16f-553817facc97 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 587.295178] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ab02fec-45e8-42f0-ad6c-da773a8a48ef {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 587.336159] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b9cc4b5a-0cee-4bd7-95bb-1ef62c7038ee {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 587.348140] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83e3ff19-5673-4c32-a2ff-05c9a3c30a86 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 587.362098] env[62499]: DEBUG nova.compute.provider_tree [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 587.448916] env[62499]: DEBUG nova.network.neutron [-] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 587.535431] env[62499]: DEBUG oslo_vmware.api [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Task: {'id': task-1335692, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.10968} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 587.535780] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Deleted the datastore file {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 587.535991] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Deleted contents of the VM from datastore datastore1 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 587.536192] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 587.536357] env[62499]: INFO nova.compute.manager [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Took 1.09 seconds to destroy the instance on the hypervisor. [ 587.536595] env[62499]: DEBUG oslo.service.loopingcall [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 587.536861] env[62499]: DEBUG nova.compute.manager [-] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 587.536861] env[62499]: DEBUG nova.network.neutron [-] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 587.633027] env[62499]: DEBUG nova.network.neutron [-] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 587.866897] env[62499]: DEBUG nova.scheduler.client.report [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 587.913037] env[62499]: ERROR nova.compute.manager [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1bf2c3a4-bfd7-4a69-a486-3f07a007056e, please check neutron logs for more information. [ 587.913037] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 587.913037] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 587.913037] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 587.913037] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 587.913037] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 587.913037] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 587.913037] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 587.913037] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 587.913037] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 587.913037] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 587.913037] env[62499]: ERROR nova.compute.manager raise self.value [ 587.913037] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 587.913037] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 587.913037] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 587.913037] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 587.913647] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 587.913647] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 587.913647] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1bf2c3a4-bfd7-4a69-a486-3f07a007056e, please check neutron logs for more information. [ 587.913647] env[62499]: ERROR nova.compute.manager [ 587.913647] env[62499]: Traceback (most recent call last): [ 587.913647] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 587.913647] env[62499]: listener.cb(fileno) [ 587.913647] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 587.913647] env[62499]: result = function(*args, **kwargs) [ 587.913647] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 587.913647] env[62499]: return func(*args, **kwargs) [ 587.913647] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 587.913647] env[62499]: raise e [ 587.913647] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 587.913647] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 587.913647] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 587.913647] env[62499]: created_port_ids = self._update_ports_for_instance( [ 587.913647] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 587.913647] env[62499]: with excutils.save_and_reraise_exception(): [ 587.913647] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 587.913647] env[62499]: self.force_reraise() [ 587.913647] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 587.913647] env[62499]: raise self.value [ 587.913647] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 587.913647] env[62499]: updated_port = self._update_port( [ 587.913647] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 587.913647] env[62499]: _ensure_no_port_binding_failure(port) [ 587.913647] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 587.913647] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 587.914533] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 1bf2c3a4-bfd7-4a69-a486-3f07a007056e, please check neutron logs for more information. [ 587.914533] env[62499]: Removing descriptor: 16 [ 587.914533] env[62499]: ERROR nova.compute.manager [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1bf2c3a4-bfd7-4a69-a486-3f07a007056e, please check neutron logs for more information. [ 587.914533] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Traceback (most recent call last): [ 587.914533] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 587.914533] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] yield resources [ 587.914533] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 587.914533] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] self.driver.spawn(context, instance, image_meta, [ 587.914533] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 587.914533] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 587.914533] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 587.914533] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] vm_ref = self.build_virtual_machine(instance, [ 587.915213] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 587.915213] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] vif_infos = vmwarevif.get_vif_info(self._session, [ 587.915213] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 587.915213] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] for vif in network_info: [ 587.915213] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 587.915213] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] return self._sync_wrapper(fn, *args, **kwargs) [ 587.915213] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 587.915213] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] self.wait() [ 587.915213] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 587.915213] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] self[:] = self._gt.wait() [ 587.915213] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 587.915213] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] return self._exit_event.wait() [ 587.915213] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 587.915617] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] result = hub.switch() [ 587.915617] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 587.915617] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] return self.greenlet.switch() [ 587.915617] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 587.915617] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] result = function(*args, **kwargs) [ 587.915617] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 587.915617] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] return func(*args, **kwargs) [ 587.915617] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 587.915617] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] raise e [ 587.915617] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 587.915617] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] nwinfo = self.network_api.allocate_for_instance( [ 587.915617] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 587.915617] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] created_port_ids = self._update_ports_for_instance( [ 587.919843] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 587.919843] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] with excutils.save_and_reraise_exception(): [ 587.919843] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 587.919843] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] self.force_reraise() [ 587.919843] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 587.919843] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] raise self.value [ 587.919843] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 587.919843] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] updated_port = self._update_port( [ 587.919843] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 587.919843] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] _ensure_no_port_binding_failure(port) [ 587.919843] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 587.919843] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] raise exception.PortBindingFailed(port_id=port['id']) [ 587.922974] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] nova.exception.PortBindingFailed: Binding failed for port 1bf2c3a4-bfd7-4a69-a486-3f07a007056e, please check neutron logs for more information. [ 587.922974] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] [ 587.922974] env[62499]: INFO nova.compute.manager [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Terminating instance [ 587.922974] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Acquiring lock "refresh_cache-7c5b9fab-bb90-4c70-b445-fe0b637dbd4a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 587.923180] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Acquired lock "refresh_cache-7c5b9fab-bb90-4c70-b445-fe0b637dbd4a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 587.923446] env[62499]: DEBUG nova.network.neutron [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 587.954348] env[62499]: DEBUG nova.compute.manager [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 587.956827] env[62499]: INFO nova.compute.manager [-] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Took 1.09 seconds to deallocate network for instance. [ 588.001670] env[62499]: DEBUG nova.virt.hardware [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 588.002036] env[62499]: DEBUG nova.virt.hardware [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 588.002167] env[62499]: DEBUG nova.virt.hardware [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 588.002344] env[62499]: DEBUG nova.virt.hardware [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 588.002485] env[62499]: DEBUG nova.virt.hardware [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 588.003168] env[62499]: DEBUG nova.virt.hardware [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 588.007599] env[62499]: DEBUG nova.virt.hardware [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 588.007599] env[62499]: DEBUG nova.virt.hardware [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 588.007599] env[62499]: DEBUG nova.virt.hardware [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 588.007599] env[62499]: DEBUG nova.virt.hardware [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 588.007599] env[62499]: DEBUG nova.virt.hardware [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 588.008068] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac3d09f5-0ee0-4207-9af6-d226978d9661 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.020846] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-347de806-bfca-4fd7-9d18-daaef676422b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.138460] env[62499]: DEBUG nova.network.neutron [-] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 588.376951] env[62499]: DEBUG oslo_concurrency.lockutils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.462s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 588.377457] env[62499]: DEBUG nova.compute.manager [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 588.380136] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 22.973s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 588.381427] env[62499]: DEBUG nova.objects.instance [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Trying to apply a migration context that does not seem to be set for this instance {{(pid=62499) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 588.470184] env[62499]: DEBUG nova.network.neutron [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 588.474444] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 588.616185] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Acquiring lock "c1cd40be-a66c-4ed9-b329-6d94509b19cf" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 588.616185] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Lock "c1cd40be-a66c-4ed9-b329-6d94509b19cf" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 588.641138] env[62499]: INFO nova.compute.manager [-] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Took 1.10 seconds to deallocate network for instance. [ 588.706780] env[62499]: DEBUG nova.network.neutron [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 588.886871] env[62499]: DEBUG nova.compute.utils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 588.890957] env[62499]: DEBUG nova.compute.manager [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 588.891142] env[62499]: DEBUG nova.network.neutron [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 589.048102] env[62499]: DEBUG nova.policy [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c92b9d37f796479ca3b6d9f0ed1789c8', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e444950a637b4fdab2edb1d62aac4be5', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 589.156412] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 589.211356] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Releasing lock "refresh_cache-7c5b9fab-bb90-4c70-b445-fe0b637dbd4a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 589.211786] env[62499]: DEBUG nova.compute.manager [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 589.211988] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 589.212761] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-bec2268e-a22f-45b6-a71a-22fad0d20770 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.225211] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-26420ea5-4781-4a28-8c2d-4775d2c72c47 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.247152] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a could not be found. [ 589.247448] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 589.247669] env[62499]: INFO nova.compute.manager [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Took 0.04 seconds to destroy the instance on the hypervisor. [ 589.248013] env[62499]: DEBUG oslo.service.loopingcall [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 589.249620] env[62499]: DEBUG nova.compute.manager [-] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 589.249726] env[62499]: DEBUG nova.network.neutron [-] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 589.252341] env[62499]: DEBUG nova.compute.manager [req-dcf21244-8a10-4a20-89b8-5c96bdb21a0c req-501b9e18-4b98-4e17-9267-e1dd71dd377f service nova] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Received event network-changed-1bf2c3a4-bfd7-4a69-a486-3f07a007056e {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 589.252554] env[62499]: DEBUG nova.compute.manager [req-dcf21244-8a10-4a20-89b8-5c96bdb21a0c req-501b9e18-4b98-4e17-9267-e1dd71dd377f service nova] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Refreshing instance network info cache due to event network-changed-1bf2c3a4-bfd7-4a69-a486-3f07a007056e. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 589.252800] env[62499]: DEBUG oslo_concurrency.lockutils [req-dcf21244-8a10-4a20-89b8-5c96bdb21a0c req-501b9e18-4b98-4e17-9267-e1dd71dd377f service nova] Acquiring lock "refresh_cache-7c5b9fab-bb90-4c70-b445-fe0b637dbd4a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 589.252994] env[62499]: DEBUG oslo_concurrency.lockutils [req-dcf21244-8a10-4a20-89b8-5c96bdb21a0c req-501b9e18-4b98-4e17-9267-e1dd71dd377f service nova] Acquired lock "refresh_cache-7c5b9fab-bb90-4c70-b445-fe0b637dbd4a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 589.253241] env[62499]: DEBUG nova.network.neutron [req-dcf21244-8a10-4a20-89b8-5c96bdb21a0c req-501b9e18-4b98-4e17-9267-e1dd71dd377f service nova] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Refreshing network info cache for port 1bf2c3a4-bfd7-4a69-a486-3f07a007056e {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 589.278585] env[62499]: DEBUG nova.network.neutron [-] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 589.392140] env[62499]: DEBUG nova.compute.manager [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 589.397571] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fa8058c3-3d0b-413a-a23c-4052b47723d2 tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.017s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 589.401336] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 20.673s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 589.401336] env[62499]: INFO nova.compute.claims [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 589.782046] env[62499]: DEBUG nova.network.neutron [-] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 589.821333] env[62499]: DEBUG nova.network.neutron [req-dcf21244-8a10-4a20-89b8-5c96bdb21a0c req-501b9e18-4b98-4e17-9267-e1dd71dd377f service nova] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 589.881190] env[62499]: DEBUG oslo_concurrency.lockutils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Acquiring lock "e654ab80-4443-4f60-9d51-6730e1907ffe" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 589.881506] env[62499]: DEBUG oslo_concurrency.lockutils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Lock "e654ab80-4443-4f60-9d51-6730e1907ffe" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 589.994311] env[62499]: DEBUG nova.network.neutron [req-dcf21244-8a10-4a20-89b8-5c96bdb21a0c req-501b9e18-4b98-4e17-9267-e1dd71dd377f service nova] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 590.057477] env[62499]: DEBUG nova.network.neutron [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Successfully created port: b5877af8-44ec-41fc-bf83-41d3e7cf35f6 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 590.285334] env[62499]: INFO nova.compute.manager [-] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Took 1.04 seconds to deallocate network for instance. [ 590.290877] env[62499]: DEBUG nova.compute.claims [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 590.291075] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 590.391925] env[62499]: ERROR nova.compute.manager [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3ecff7db-5412-4e0b-bc53-9a1924aa0f90, please check neutron logs for more information. [ 590.391925] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 590.391925] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 590.391925] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 590.391925] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 590.391925] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 590.391925] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 590.391925] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 590.391925] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 590.391925] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 590.391925] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 590.391925] env[62499]: ERROR nova.compute.manager raise self.value [ 590.391925] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 590.391925] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 590.391925] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 590.391925] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 590.392547] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 590.392547] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 590.392547] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3ecff7db-5412-4e0b-bc53-9a1924aa0f90, please check neutron logs for more information. [ 590.392547] env[62499]: ERROR nova.compute.manager [ 590.393068] env[62499]: Traceback (most recent call last): [ 590.393215] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 590.393215] env[62499]: listener.cb(fileno) [ 590.393328] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 590.393328] env[62499]: result = function(*args, **kwargs) [ 590.393401] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 590.393401] env[62499]: return func(*args, **kwargs) [ 590.393623] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 590.393623] env[62499]: raise e [ 590.393909] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 590.393909] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 590.393995] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 590.393995] env[62499]: created_port_ids = self._update_ports_for_instance( [ 590.394088] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 590.394088] env[62499]: with excutils.save_and_reraise_exception(): [ 590.394164] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 590.394164] env[62499]: self.force_reraise() [ 590.394261] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 590.394261] env[62499]: raise self.value [ 590.395239] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 590.395239] env[62499]: updated_port = self._update_port( [ 590.395424] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 590.395424] env[62499]: _ensure_no_port_binding_failure(port) [ 590.395500] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 590.395500] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 590.395591] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 3ecff7db-5412-4e0b-bc53-9a1924aa0f90, please check neutron logs for more information. [ 590.395644] env[62499]: Removing descriptor: 15 [ 590.396936] env[62499]: ERROR nova.compute.manager [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3ecff7db-5412-4e0b-bc53-9a1924aa0f90, please check neutron logs for more information. [ 590.396936] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Traceback (most recent call last): [ 590.396936] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 590.396936] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] yield resources [ 590.396936] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 590.396936] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] self.driver.spawn(context, instance, image_meta, [ 590.396936] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 590.396936] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] self._vmops.spawn(context, instance, image_meta, injected_files, [ 590.396936] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 590.396936] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] vm_ref = self.build_virtual_machine(instance, [ 590.396936] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 590.397356] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] vif_infos = vmwarevif.get_vif_info(self._session, [ 590.397356] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 590.397356] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] for vif in network_info: [ 590.397356] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 590.397356] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] return self._sync_wrapper(fn, *args, **kwargs) [ 590.397356] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 590.397356] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] self.wait() [ 590.397356] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 590.397356] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] self[:] = self._gt.wait() [ 590.397356] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 590.397356] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] return self._exit_event.wait() [ 590.397356] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 590.397356] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] result = hub.switch() [ 590.397739] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 590.397739] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] return self.greenlet.switch() [ 590.397739] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 590.397739] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] result = function(*args, **kwargs) [ 590.397739] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 590.397739] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] return func(*args, **kwargs) [ 590.397739] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 590.397739] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] raise e [ 590.397739] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 590.397739] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] nwinfo = self.network_api.allocate_for_instance( [ 590.397739] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 590.397739] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] created_port_ids = self._update_ports_for_instance( [ 590.397739] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 590.398161] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] with excutils.save_and_reraise_exception(): [ 590.398161] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 590.398161] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] self.force_reraise() [ 590.398161] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 590.398161] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] raise self.value [ 590.398161] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 590.398161] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] updated_port = self._update_port( [ 590.398161] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 590.398161] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] _ensure_no_port_binding_failure(port) [ 590.398161] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 590.398161] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] raise exception.PortBindingFailed(port_id=port['id']) [ 590.398161] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] nova.exception.PortBindingFailed: Binding failed for port 3ecff7db-5412-4e0b-bc53-9a1924aa0f90, please check neutron logs for more information. [ 590.398161] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] [ 590.400660] env[62499]: INFO nova.compute.manager [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Terminating instance [ 590.408019] env[62499]: DEBUG oslo_concurrency.lockutils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Acquiring lock "refresh_cache-3706243f-4768-4b44-9a5b-730bf2b18974" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 590.408019] env[62499]: DEBUG oslo_concurrency.lockutils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Acquired lock "refresh_cache-3706243f-4768-4b44-9a5b-730bf2b18974" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 590.408019] env[62499]: DEBUG nova.network.neutron [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 590.408019] env[62499]: DEBUG nova.compute.manager [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 590.443207] env[62499]: DEBUG nova.virt.hardware [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 590.445742] env[62499]: DEBUG nova.virt.hardware [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 590.445742] env[62499]: DEBUG nova.virt.hardware [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 590.445742] env[62499]: DEBUG nova.virt.hardware [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 590.445742] env[62499]: DEBUG nova.virt.hardware [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 590.445742] env[62499]: DEBUG nova.virt.hardware [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 590.445944] env[62499]: DEBUG nova.virt.hardware [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 590.445944] env[62499]: DEBUG nova.virt.hardware [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 590.445944] env[62499]: DEBUG nova.virt.hardware [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 590.445944] env[62499]: DEBUG nova.virt.hardware [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 590.445944] env[62499]: DEBUG nova.virt.hardware [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 590.446622] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-318e74ab-3508-4ebd-9a7d-2f2fff5cb120 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 590.464826] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d6dd92c-371e-4744-976a-37d514ec7d93 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 590.497621] env[62499]: DEBUG oslo_concurrency.lockutils [req-dcf21244-8a10-4a20-89b8-5c96bdb21a0c req-501b9e18-4b98-4e17-9267-e1dd71dd377f service nova] Releasing lock "refresh_cache-7c5b9fab-bb90-4c70-b445-fe0b637dbd4a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 590.498303] env[62499]: DEBUG nova.compute.manager [req-dcf21244-8a10-4a20-89b8-5c96bdb21a0c req-501b9e18-4b98-4e17-9267-e1dd71dd377f service nova] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Received event network-vif-deleted-1bf2c3a4-bfd7-4a69-a486-3f07a007056e {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 590.745362] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5fc9a819-2740-4445-b8e0-cc0add0354bb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 590.753516] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8328eb76-d3f0-4eda-bed2-65ca5b9a7443 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 590.786071] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5767262-d5c0-4eaf-9863-5de1b1e59eaf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 590.795161] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-646207ed-ec94-41e8-8338-90751f0e09f4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 590.808204] env[62499]: DEBUG nova.compute.provider_tree [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 590.941255] env[62499]: DEBUG nova.network.neutron [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 591.113289] env[62499]: DEBUG nova.network.neutron [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 591.139745] env[62499]: DEBUG oslo_concurrency.lockutils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Acquiring lock "28832fde-2434-4d5a-ae62-550b8c0eb829" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 591.139974] env[62499]: DEBUG oslo_concurrency.lockutils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Lock "28832fde-2434-4d5a-ae62-550b8c0eb829" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 591.315582] env[62499]: DEBUG nova.scheduler.client.report [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 591.588819] env[62499]: ERROR nova.compute.manager [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b5877af8-44ec-41fc-bf83-41d3e7cf35f6, please check neutron logs for more information. [ 591.588819] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 591.588819] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 591.588819] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 591.588819] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 591.588819] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 591.588819] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 591.588819] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 591.588819] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 591.588819] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 591.588819] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 591.588819] env[62499]: ERROR nova.compute.manager raise self.value [ 591.588819] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 591.588819] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 591.588819] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 591.588819] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 591.589361] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 591.589361] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 591.589361] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b5877af8-44ec-41fc-bf83-41d3e7cf35f6, please check neutron logs for more information. [ 591.589361] env[62499]: ERROR nova.compute.manager [ 591.589361] env[62499]: Traceback (most recent call last): [ 591.589361] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 591.589361] env[62499]: listener.cb(fileno) [ 591.589361] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 591.589361] env[62499]: result = function(*args, **kwargs) [ 591.589361] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 591.589361] env[62499]: return func(*args, **kwargs) [ 591.589361] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 591.589361] env[62499]: raise e [ 591.589361] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 591.589361] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 591.589361] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 591.589361] env[62499]: created_port_ids = self._update_ports_for_instance( [ 591.589361] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 591.589361] env[62499]: with excutils.save_and_reraise_exception(): [ 591.589361] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 591.589361] env[62499]: self.force_reraise() [ 591.589361] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 591.589361] env[62499]: raise self.value [ 591.589361] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 591.589361] env[62499]: updated_port = self._update_port( [ 591.589361] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 591.589361] env[62499]: _ensure_no_port_binding_failure(port) [ 591.589361] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 591.589361] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 591.590561] env[62499]: nova.exception.PortBindingFailed: Binding failed for port b5877af8-44ec-41fc-bf83-41d3e7cf35f6, please check neutron logs for more information. [ 591.590561] env[62499]: Removing descriptor: 16 [ 591.590561] env[62499]: ERROR nova.compute.manager [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b5877af8-44ec-41fc-bf83-41d3e7cf35f6, please check neutron logs for more information. [ 591.590561] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Traceback (most recent call last): [ 591.590561] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 591.590561] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] yield resources [ 591.590561] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 591.590561] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] self.driver.spawn(context, instance, image_meta, [ 591.590561] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 591.590561] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 591.590561] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 591.590561] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] vm_ref = self.build_virtual_machine(instance, [ 591.590997] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 591.590997] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] vif_infos = vmwarevif.get_vif_info(self._session, [ 591.590997] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 591.590997] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] for vif in network_info: [ 591.590997] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 591.590997] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] return self._sync_wrapper(fn, *args, **kwargs) [ 591.590997] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 591.590997] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] self.wait() [ 591.590997] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 591.590997] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] self[:] = self._gt.wait() [ 591.590997] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 591.590997] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] return self._exit_event.wait() [ 591.590997] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 591.592027] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] result = hub.switch() [ 591.592027] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 591.592027] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] return self.greenlet.switch() [ 591.592027] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 591.592027] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] result = function(*args, **kwargs) [ 591.592027] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 591.592027] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] return func(*args, **kwargs) [ 591.592027] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 591.592027] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] raise e [ 591.592027] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 591.592027] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] nwinfo = self.network_api.allocate_for_instance( [ 591.592027] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 591.592027] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] created_port_ids = self._update_ports_for_instance( [ 591.592862] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 591.592862] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] with excutils.save_and_reraise_exception(): [ 591.592862] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 591.592862] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] self.force_reraise() [ 591.592862] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 591.592862] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] raise self.value [ 591.592862] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 591.592862] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] updated_port = self._update_port( [ 591.592862] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 591.592862] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] _ensure_no_port_binding_failure(port) [ 591.592862] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 591.592862] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] raise exception.PortBindingFailed(port_id=port['id']) [ 591.593292] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] nova.exception.PortBindingFailed: Binding failed for port b5877af8-44ec-41fc-bf83-41d3e7cf35f6, please check neutron logs for more information. [ 591.593292] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] [ 591.593292] env[62499]: INFO nova.compute.manager [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Terminating instance [ 591.595687] env[62499]: DEBUG oslo_concurrency.lockutils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "refresh_cache-9c7433a1-9d5b-4ef2-ad65-f1217f0e862c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 591.595687] env[62499]: DEBUG oslo_concurrency.lockutils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquired lock "refresh_cache-9c7433a1-9d5b-4ef2-ad65-f1217f0e862c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 591.596095] env[62499]: DEBUG nova.network.neutron [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 591.619321] env[62499]: DEBUG oslo_concurrency.lockutils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Releasing lock "refresh_cache-3706243f-4768-4b44-9a5b-730bf2b18974" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 591.619721] env[62499]: DEBUG nova.compute.manager [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 591.619988] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 591.620346] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-02e89e3a-19d6-44df-b3c7-7168b89f3897 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.632058] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a52040a4-b39e-46b1-bc96-f771eef3c214 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.657379] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3706243f-4768-4b44-9a5b-730bf2b18974 could not be found. [ 591.657857] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 591.658218] env[62499]: INFO nova.compute.manager [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Took 0.04 seconds to destroy the instance on the hypervisor. [ 591.658466] env[62499]: DEBUG oslo.service.loopingcall [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 591.659114] env[62499]: DEBUG nova.compute.manager [-] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 591.659261] env[62499]: DEBUG nova.network.neutron [-] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 591.679018] env[62499]: DEBUG nova.network.neutron [-] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 591.707842] env[62499]: DEBUG nova.compute.manager [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Received event network-changed-3ecff7db-5412-4e0b-bc53-9a1924aa0f90 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 591.708613] env[62499]: DEBUG nova.compute.manager [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Refreshing instance network info cache due to event network-changed-3ecff7db-5412-4e0b-bc53-9a1924aa0f90. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 591.708867] env[62499]: DEBUG oslo_concurrency.lockutils [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] Acquiring lock "refresh_cache-3706243f-4768-4b44-9a5b-730bf2b18974" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 591.709320] env[62499]: DEBUG oslo_concurrency.lockutils [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] Acquired lock "refresh_cache-3706243f-4768-4b44-9a5b-730bf2b18974" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 591.709320] env[62499]: DEBUG nova.network.neutron [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Refreshing network info cache for port 3ecff7db-5412-4e0b-bc53-9a1924aa0f90 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 591.821245] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.421s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 591.821245] env[62499]: DEBUG nova.compute.manager [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 591.824957] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 19.881s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 592.140175] env[62499]: DEBUG nova.network.neutron [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 592.181527] env[62499]: DEBUG nova.network.neutron [-] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 592.245452] env[62499]: DEBUG nova.network.neutron [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 592.329231] env[62499]: DEBUG nova.compute.utils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 592.333902] env[62499]: DEBUG nova.compute.manager [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 592.334249] env[62499]: DEBUG nova.network.neutron [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 592.389425] env[62499]: DEBUG nova.network.neutron [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 592.466284] env[62499]: DEBUG nova.network.neutron [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 592.495807] env[62499]: DEBUG nova.policy [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '9ad60a9ead734f08aa163c15c9a6b81d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9c30565c40a845588906f604e8cb0e00', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 592.685901] env[62499]: INFO nova.compute.manager [-] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Took 1.03 seconds to deallocate network for instance. [ 592.689604] env[62499]: DEBUG nova.compute.claims [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 592.689863] env[62499]: DEBUG oslo_concurrency.lockutils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 592.743434] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b562ce70-0fcc-4dce-9e6f-b9f9ad0310ad {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.755574] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95fa063c-d48d-426c-8630-3f0112792544 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.793515] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fcb113db-363d-4688-9852-739cd85adb3f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.802302] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eac0c2d9-4139-407a-ac0c-226b552760e5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.817623] env[62499]: DEBUG nova.compute.provider_tree [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 592.834432] env[62499]: DEBUG nova.compute.manager [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 592.897472] env[62499]: DEBUG oslo_concurrency.lockutils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Releasing lock "refresh_cache-9c7433a1-9d5b-4ef2-ad65-f1217f0e862c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 592.897891] env[62499]: DEBUG nova.compute.manager [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 592.898139] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 592.898822] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-110f219c-26a2-41f3-9f21-72adfd8c7b4a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.906932] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-03a0e068-1770-468b-a2ed-f0174a800f30 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.939024] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c could not be found. [ 592.939024] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 592.939024] env[62499]: INFO nova.compute.manager [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Took 0.04 seconds to destroy the instance on the hypervisor. [ 592.939024] env[62499]: DEBUG oslo.service.loopingcall [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 592.939024] env[62499]: DEBUG nova.compute.manager [-] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 592.939024] env[62499]: DEBUG nova.network.neutron [-] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 592.967566] env[62499]: DEBUG oslo_concurrency.lockutils [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] Releasing lock "refresh_cache-3706243f-4768-4b44-9a5b-730bf2b18974" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 592.967683] env[62499]: DEBUG nova.compute.manager [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Received event network-vif-deleted-3ecff7db-5412-4e0b-bc53-9a1924aa0f90 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 592.967825] env[62499]: DEBUG nova.compute.manager [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Received event network-changed-b5877af8-44ec-41fc-bf83-41d3e7cf35f6 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 592.967992] env[62499]: DEBUG nova.compute.manager [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Refreshing instance network info cache due to event network-changed-b5877af8-44ec-41fc-bf83-41d3e7cf35f6. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 592.968214] env[62499]: DEBUG oslo_concurrency.lockutils [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] Acquiring lock "refresh_cache-9c7433a1-9d5b-4ef2-ad65-f1217f0e862c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 592.968355] env[62499]: DEBUG oslo_concurrency.lockutils [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] Acquired lock "refresh_cache-9c7433a1-9d5b-4ef2-ad65-f1217f0e862c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 592.968513] env[62499]: DEBUG nova.network.neutron [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Refreshing network info cache for port b5877af8-44ec-41fc-bf83-41d3e7cf35f6 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 592.980860] env[62499]: DEBUG nova.network.neutron [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Successfully created port: 10bc8270-f0af-4ac1-984d-dedf660a797d {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 592.986996] env[62499]: DEBUG nova.network.neutron [-] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 593.321085] env[62499]: DEBUG nova.scheduler.client.report [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 593.375628] env[62499]: DEBUG nova.network.neutron [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Successfully created port: 2a798717-3ccd-471e-a8c5-2c85e1ea91dc {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 593.488458] env[62499]: DEBUG nova.network.neutron [-] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 593.501182] env[62499]: DEBUG nova.network.neutron [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 593.675583] env[62499]: DEBUG nova.network.neutron [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 593.826901] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.004s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 593.827555] env[62499]: ERROR nova.compute.manager [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 42bcc024-8460-4983-9bc4-d3545aa63650, please check neutron logs for more information. [ 593.827555] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Traceback (most recent call last): [ 593.827555] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 593.827555] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] self.driver.spawn(context, instance, image_meta, [ 593.827555] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 593.827555] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] self._vmops.spawn(context, instance, image_meta, injected_files, [ 593.827555] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 593.827555] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] vm_ref = self.build_virtual_machine(instance, [ 593.827555] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 593.827555] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] vif_infos = vmwarevif.get_vif_info(self._session, [ 593.827555] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 593.828035] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] for vif in network_info: [ 593.828035] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 593.828035] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] return self._sync_wrapper(fn, *args, **kwargs) [ 593.828035] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 593.828035] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] self.wait() [ 593.828035] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 593.828035] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] self[:] = self._gt.wait() [ 593.828035] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 593.828035] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] return self._exit_event.wait() [ 593.828035] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 593.828035] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] result = hub.switch() [ 593.828035] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 593.828035] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] return self.greenlet.switch() [ 593.828519] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 593.828519] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] result = function(*args, **kwargs) [ 593.828519] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 593.828519] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] return func(*args, **kwargs) [ 593.828519] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 593.828519] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] raise e [ 593.828519] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 593.828519] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] nwinfo = self.network_api.allocate_for_instance( [ 593.828519] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 593.828519] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] created_port_ids = self._update_ports_for_instance( [ 593.828519] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 593.828519] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] with excutils.save_and_reraise_exception(): [ 593.828519] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 593.829075] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] self.force_reraise() [ 593.829075] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 593.829075] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] raise self.value [ 593.829075] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 593.829075] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] updated_port = self._update_port( [ 593.829075] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 593.829075] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] _ensure_no_port_binding_failure(port) [ 593.829075] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 593.829075] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] raise exception.PortBindingFailed(port_id=port['id']) [ 593.829075] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] nova.exception.PortBindingFailed: Binding failed for port 42bcc024-8460-4983-9bc4-d3545aa63650, please check neutron logs for more information. [ 593.829075] env[62499]: ERROR nova.compute.manager [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] [ 593.829504] env[62499]: DEBUG nova.compute.utils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Binding failed for port 42bcc024-8460-4983-9bc4-d3545aa63650, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 593.829504] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 21.065s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 593.830980] env[62499]: INFO nova.compute.claims [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 593.834229] env[62499]: DEBUG nova.compute.manager [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Build of instance 2b7122b2-136d-4e0c-97eb-155b33905436 was re-scheduled: Binding failed for port 42bcc024-8460-4983-9bc4-d3545aa63650, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 593.835123] env[62499]: DEBUG nova.compute.manager [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 593.835686] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquiring lock "refresh_cache-2b7122b2-136d-4e0c-97eb-155b33905436" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 593.835686] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquired lock "refresh_cache-2b7122b2-136d-4e0c-97eb-155b33905436" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 593.835798] env[62499]: DEBUG nova.network.neutron [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 593.846443] env[62499]: DEBUG nova.compute.manager [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 593.881247] env[62499]: DEBUG nova.virt.hardware [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 593.881571] env[62499]: DEBUG nova.virt.hardware [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 593.881651] env[62499]: DEBUG nova.virt.hardware [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 593.881886] env[62499]: DEBUG nova.virt.hardware [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 593.886699] env[62499]: DEBUG nova.virt.hardware [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 593.886769] env[62499]: DEBUG nova.virt.hardware [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 593.887022] env[62499]: DEBUG nova.virt.hardware [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 593.887197] env[62499]: DEBUG nova.virt.hardware [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 593.887371] env[62499]: DEBUG nova.virt.hardware [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 593.887549] env[62499]: DEBUG nova.virt.hardware [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 593.887730] env[62499]: DEBUG nova.virt.hardware [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 593.890083] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7c0184e-d63e-4831-92e0-95369835c886 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 593.900021] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d07ba5c-cc7c-4c9a-a071-b5db192fe28c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 593.960544] env[62499]: DEBUG nova.network.neutron [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Successfully created port: f0476d40-0c5a-4527-8610-b9dc334b3738 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 593.991423] env[62499]: INFO nova.compute.manager [-] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Took 1.05 seconds to deallocate network for instance. [ 593.993356] env[62499]: DEBUG nova.compute.claims [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 593.993547] env[62499]: DEBUG oslo_concurrency.lockutils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 594.061733] env[62499]: DEBUG nova.compute.manager [req-c084e71d-b70c-4a67-8d34-0eea137d5b25 req-fa93acb3-8df6-434d-b74b-2dadae1b1ff5 service nova] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Received event network-vif-deleted-b5877af8-44ec-41fc-bf83-41d3e7cf35f6 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 594.181017] env[62499]: DEBUG oslo_concurrency.lockutils [req-f581bafa-4d9f-48fb-bc36-492cf98adaf5 req-40de4bd0-67e3-46d1-b346-79f905e4a6e1 service nova] Releasing lock "refresh_cache-9c7433a1-9d5b-4ef2-ad65-f1217f0e862c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 594.363648] env[62499]: DEBUG nova.network.neutron [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 594.475267] env[62499]: DEBUG nova.network.neutron [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 594.980600] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Releasing lock "refresh_cache-2b7122b2-136d-4e0c-97eb-155b33905436" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 594.980600] env[62499]: DEBUG nova.compute.manager [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 594.980600] env[62499]: DEBUG nova.compute.manager [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 594.980600] env[62499]: DEBUG nova.network.neutron [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 595.009240] env[62499]: DEBUG nova.network.neutron [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 595.200271] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd06e8b9-857d-4402-8132-f9582df605c1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 595.208652] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e48e129e-a7e5-4f08-ba45-ff00e5803ecc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 595.242677] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f3969fd8-7181-48de-a648-ce568a8ad19a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 595.250398] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e4ef0ecd-ca2c-4f9e-9dc1-7ada1e0245b9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 595.267514] env[62499]: DEBUG nova.compute.provider_tree [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 595.445157] env[62499]: DEBUG oslo_concurrency.lockutils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Acquiring lock "873731fb-8d97-41e7-830b-b9ae4281a737" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 595.445396] env[62499]: DEBUG oslo_concurrency.lockutils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Lock "873731fb-8d97-41e7-830b-b9ae4281a737" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 595.512272] env[62499]: DEBUG nova.network.neutron [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 595.771044] env[62499]: DEBUG nova.scheduler.client.report [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 596.022392] env[62499]: INFO nova.compute.manager [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 2b7122b2-136d-4e0c-97eb-155b33905436] Took 1.04 seconds to deallocate network for instance. [ 596.023546] env[62499]: ERROR nova.compute.manager [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 10bc8270-f0af-4ac1-984d-dedf660a797d, please check neutron logs for more information. [ 596.023546] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 596.023546] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 596.023546] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 596.023546] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 596.023546] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 596.023546] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 596.023546] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 596.023546] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 596.023546] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 596.023546] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 596.023546] env[62499]: ERROR nova.compute.manager raise self.value [ 596.023546] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 596.023546] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 596.023546] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 596.023546] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 596.024546] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 596.024546] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 596.024546] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 10bc8270-f0af-4ac1-984d-dedf660a797d, please check neutron logs for more information. [ 596.024546] env[62499]: ERROR nova.compute.manager [ 596.024546] env[62499]: Traceback (most recent call last): [ 596.024546] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 596.024546] env[62499]: listener.cb(fileno) [ 596.024546] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 596.024546] env[62499]: result = function(*args, **kwargs) [ 596.024546] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 596.024546] env[62499]: return func(*args, **kwargs) [ 596.024546] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 596.024546] env[62499]: raise e [ 596.024546] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 596.024546] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 596.024546] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 596.024546] env[62499]: created_port_ids = self._update_ports_for_instance( [ 596.024546] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 596.024546] env[62499]: with excutils.save_and_reraise_exception(): [ 596.024546] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 596.024546] env[62499]: self.force_reraise() [ 596.024546] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 596.024546] env[62499]: raise self.value [ 596.024546] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 596.024546] env[62499]: updated_port = self._update_port( [ 596.024546] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 596.024546] env[62499]: _ensure_no_port_binding_failure(port) [ 596.024546] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 596.024546] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 596.026825] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 10bc8270-f0af-4ac1-984d-dedf660a797d, please check neutron logs for more information. [ 596.026825] env[62499]: Removing descriptor: 17 [ 596.026825] env[62499]: ERROR nova.compute.manager [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 10bc8270-f0af-4ac1-984d-dedf660a797d, please check neutron logs for more information. [ 596.026825] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Traceback (most recent call last): [ 596.026825] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 596.026825] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] yield resources [ 596.026825] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 596.026825] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] self.driver.spawn(context, instance, image_meta, [ 596.026825] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 596.026825] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] self._vmops.spawn(context, instance, image_meta, injected_files, [ 596.026825] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 596.026825] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] vm_ref = self.build_virtual_machine(instance, [ 596.030637] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 596.030637] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] vif_infos = vmwarevif.get_vif_info(self._session, [ 596.030637] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 596.030637] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] for vif in network_info: [ 596.030637] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 596.030637] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] return self._sync_wrapper(fn, *args, **kwargs) [ 596.030637] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 596.030637] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] self.wait() [ 596.030637] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 596.030637] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] self[:] = self._gt.wait() [ 596.030637] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 596.030637] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] return self._exit_event.wait() [ 596.030637] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 596.032084] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] result = hub.switch() [ 596.032084] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 596.032084] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] return self.greenlet.switch() [ 596.032084] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 596.032084] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] result = function(*args, **kwargs) [ 596.032084] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 596.032084] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] return func(*args, **kwargs) [ 596.032084] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 596.032084] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] raise e [ 596.032084] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 596.032084] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] nwinfo = self.network_api.allocate_for_instance( [ 596.032084] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 596.032084] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] created_port_ids = self._update_ports_for_instance( [ 596.033804] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 596.033804] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] with excutils.save_and_reraise_exception(): [ 596.033804] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 596.033804] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] self.force_reraise() [ 596.033804] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 596.033804] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] raise self.value [ 596.033804] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 596.033804] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] updated_port = self._update_port( [ 596.033804] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 596.033804] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] _ensure_no_port_binding_failure(port) [ 596.033804] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 596.033804] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] raise exception.PortBindingFailed(port_id=port['id']) [ 596.034501] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] nova.exception.PortBindingFailed: Binding failed for port 10bc8270-f0af-4ac1-984d-dedf660a797d, please check neutron logs for more information. [ 596.034501] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] [ 596.034501] env[62499]: INFO nova.compute.manager [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Terminating instance [ 596.034501] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Acquiring lock "refresh_cache-0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 596.034501] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Acquired lock "refresh_cache-0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 596.034501] env[62499]: DEBUG nova.network.neutron [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 596.147268] env[62499]: DEBUG nova.compute.manager [req-d801115c-6557-4ff2-a412-bc5b681407af req-67b66ebb-fe18-4a0f-99c3-9074eae7e04f service nova] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Received event network-changed-10bc8270-f0af-4ac1-984d-dedf660a797d {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 596.147446] env[62499]: DEBUG nova.compute.manager [req-d801115c-6557-4ff2-a412-bc5b681407af req-67b66ebb-fe18-4a0f-99c3-9074eae7e04f service nova] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Refreshing instance network info cache due to event network-changed-10bc8270-f0af-4ac1-984d-dedf660a797d. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 596.147628] env[62499]: DEBUG oslo_concurrency.lockutils [req-d801115c-6557-4ff2-a412-bc5b681407af req-67b66ebb-fe18-4a0f-99c3-9074eae7e04f service nova] Acquiring lock "refresh_cache-0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 596.282155] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.451s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 596.282155] env[62499]: DEBUG nova.compute.manager [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 596.284146] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 21.142s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 596.580028] env[62499]: DEBUG nova.network.neutron [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 596.710015] env[62499]: DEBUG nova.network.neutron [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 596.792311] env[62499]: DEBUG nova.compute.utils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 596.798251] env[62499]: DEBUG nova.compute.manager [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 596.799342] env[62499]: DEBUG nova.network.neutron [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 596.888842] env[62499]: DEBUG nova.policy [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c07a7f3a5cce4d9a9ec16ec04c0cf198', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'a3c7f542e2214aeb9c76e9523151af1e', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 597.055590] env[62499]: INFO nova.scheduler.client.report [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Deleted allocations for instance 2b7122b2-136d-4e0c-97eb-155b33905436 [ 597.215607] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Releasing lock "refresh_cache-0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 597.216073] env[62499]: DEBUG nova.compute.manager [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 597.216298] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 597.216607] env[62499]: DEBUG oslo_concurrency.lockutils [req-d801115c-6557-4ff2-a412-bc5b681407af req-67b66ebb-fe18-4a0f-99c3-9074eae7e04f service nova] Acquired lock "refresh_cache-0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 597.216778] env[62499]: DEBUG nova.network.neutron [req-d801115c-6557-4ff2-a412-bc5b681407af req-67b66ebb-fe18-4a0f-99c3-9074eae7e04f service nova] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Refreshing network info cache for port 10bc8270-f0af-4ac1-984d-dedf660a797d {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 597.217836] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-0bbea6ff-0cff-41de-a8da-6cc98657b8ee {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 597.229156] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa8afdf3-e01e-40ff-a39c-c9cc445daf68 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 597.236922] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-afe42709-4714-48f9-8e0b-5f265368889a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 597.251902] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-31b9b642-b3ac-4d8a-9486-489518ebc60f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 597.288625] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed821dfb-e277-49b6-8138-2f5f3de985ba {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 597.290619] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733 could not be found. [ 597.290837] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 597.291023] env[62499]: INFO nova.compute.manager [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Took 0.07 seconds to destroy the instance on the hypervisor. [ 597.291337] env[62499]: DEBUG oslo.service.loopingcall [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 597.291483] env[62499]: DEBUG nova.compute.manager [-] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 597.291580] env[62499]: DEBUG nova.network.neutron [-] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 597.298262] env[62499]: DEBUG nova.compute.manager [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 597.302364] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3fc6a9dd-1224-49c8-b03e-010efc1ca03f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 597.320114] env[62499]: DEBUG nova.compute.provider_tree [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 597.574216] env[62499]: DEBUG nova.network.neutron [-] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 597.578118] env[62499]: DEBUG oslo_concurrency.lockutils [None req-01bcd023-3223-4eff-b64d-5aa8ff88166a tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "2b7122b2-136d-4e0c-97eb-155b33905436" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 47.320s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 597.585741] env[62499]: DEBUG nova.network.neutron [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Successfully created port: 6eb83bc1-e5fd-4aba-bb63-fc8d979dbabf {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 597.821390] env[62499]: DEBUG nova.scheduler.client.report [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 597.978008] env[62499]: DEBUG nova.network.neutron [req-d801115c-6557-4ff2-a412-bc5b681407af req-67b66ebb-fe18-4a0f-99c3-9074eae7e04f service nova] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 598.079615] env[62499]: DEBUG nova.compute.manager [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 598.245441] env[62499]: DEBUG nova.network.neutron [req-d801115c-6557-4ff2-a412-bc5b681407af req-67b66ebb-fe18-4a0f-99c3-9074eae7e04f service nova] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 598.313997] env[62499]: DEBUG nova.compute.manager [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 598.330815] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.047s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 598.331506] env[62499]: ERROR nova.compute.manager [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 60f230d6-abe8-45a2-896b-081db4709502, please check neutron logs for more information. [ 598.331506] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Traceback (most recent call last): [ 598.331506] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 598.331506] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] self.driver.spawn(context, instance, image_meta, [ 598.331506] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 598.331506] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 598.331506] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 598.331506] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] vm_ref = self.build_virtual_machine(instance, [ 598.331506] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 598.331506] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] vif_infos = vmwarevif.get_vif_info(self._session, [ 598.331506] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 598.333478] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] for vif in network_info: [ 598.333478] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 598.333478] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] return self._sync_wrapper(fn, *args, **kwargs) [ 598.333478] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 598.333478] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] self.wait() [ 598.333478] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 598.333478] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] self[:] = self._gt.wait() [ 598.333478] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 598.333478] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] return self._exit_event.wait() [ 598.333478] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 598.333478] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] result = hub.switch() [ 598.333478] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 598.333478] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] return self.greenlet.switch() [ 598.335232] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 598.335232] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] result = function(*args, **kwargs) [ 598.335232] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 598.335232] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] return func(*args, **kwargs) [ 598.335232] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 598.335232] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] raise e [ 598.335232] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 598.335232] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] nwinfo = self.network_api.allocate_for_instance( [ 598.335232] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 598.335232] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] created_port_ids = self._update_ports_for_instance( [ 598.335232] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 598.335232] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] with excutils.save_and_reraise_exception(): [ 598.335232] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 598.335721] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] self.force_reraise() [ 598.335721] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 598.335721] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] raise self.value [ 598.335721] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 598.335721] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] updated_port = self._update_port( [ 598.335721] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 598.335721] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] _ensure_no_port_binding_failure(port) [ 598.335721] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 598.335721] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] raise exception.PortBindingFailed(port_id=port['id']) [ 598.335721] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] nova.exception.PortBindingFailed: Binding failed for port 60f230d6-abe8-45a2-896b-081db4709502, please check neutron logs for more information. [ 598.335721] env[62499]: ERROR nova.compute.manager [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] [ 598.336103] env[62499]: DEBUG nova.compute.utils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Binding failed for port 60f230d6-abe8-45a2-896b-081db4709502, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 598.336103] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.126s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 598.336243] env[62499]: INFO nova.compute.claims [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 598.342038] env[62499]: DEBUG nova.compute.manager [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Build of instance 76cf7f28-7413-4b06-bd30-ae22bbc2999e was re-scheduled: Binding failed for port 60f230d6-abe8-45a2-896b-081db4709502, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 598.342038] env[62499]: DEBUG nova.compute.manager [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 598.342038] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Acquiring lock "refresh_cache-76cf7f28-7413-4b06-bd30-ae22bbc2999e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 598.342038] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Acquired lock "refresh_cache-76cf7f28-7413-4b06-bd30-ae22bbc2999e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 598.342301] env[62499]: DEBUG nova.network.neutron [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 598.353318] env[62499]: DEBUG nova.virt.hardware [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 598.356867] env[62499]: DEBUG nova.virt.hardware [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 598.356867] env[62499]: DEBUG nova.virt.hardware [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 598.356867] env[62499]: DEBUG nova.virt.hardware [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 598.356867] env[62499]: DEBUG nova.virt.hardware [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 598.356867] env[62499]: DEBUG nova.virt.hardware [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 598.357152] env[62499]: DEBUG nova.virt.hardware [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 598.357152] env[62499]: DEBUG nova.virt.hardware [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 598.357152] env[62499]: DEBUG nova.virt.hardware [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 598.357152] env[62499]: DEBUG nova.virt.hardware [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 598.357152] env[62499]: DEBUG nova.virt.hardware [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 598.357334] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-101a7c47-a730-42d5-9692-c2d0c9693d17 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 598.375120] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96eba534-99cd-4a7d-bda9-748ad8266e6d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 598.533224] env[62499]: DEBUG oslo_concurrency.lockutils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Acquiring lock "9ca13604-2b1b-4d47-8a23-8137384a87a6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 598.533224] env[62499]: DEBUG oslo_concurrency.lockutils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Lock "9ca13604-2b1b-4d47-8a23-8137384a87a6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 598.612415] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 598.749240] env[62499]: DEBUG oslo_concurrency.lockutils [req-d801115c-6557-4ff2-a412-bc5b681407af req-67b66ebb-fe18-4a0f-99c3-9074eae7e04f service nova] Releasing lock "refresh_cache-0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 598.875533] env[62499]: DEBUG nova.network.neutron [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 598.933721] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Acquiring lock "3b5fb42c-8a81-4575-ba8f-424a101d11ae" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 598.934073] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Lock "3b5fb42c-8a81-4575-ba8f-424a101d11ae" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 598.961930] env[62499]: DEBUG nova.network.neutron [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 599.252840] env[62499]: DEBUG nova.network.neutron [-] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 599.340676] env[62499]: DEBUG nova.compute.manager [req-555d9c9d-e827-4310-955e-bf50cb8b4dbf req-19545351-380d-4602-a376-0536e2b2932b service nova] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Received event network-vif-deleted-10bc8270-f0af-4ac1-984d-dedf660a797d {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 599.466232] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Releasing lock "refresh_cache-76cf7f28-7413-4b06-bd30-ae22bbc2999e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 599.466232] env[62499]: DEBUG nova.compute.manager [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 599.466601] env[62499]: DEBUG nova.compute.manager [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 599.466925] env[62499]: DEBUG nova.network.neutron [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 599.495290] env[62499]: DEBUG nova.network.neutron [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 599.756238] env[62499]: INFO nova.compute.manager [-] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Took 2.46 seconds to deallocate network for instance. [ 599.758920] env[62499]: DEBUG nova.compute.claims [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 599.759048] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 599.783129] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45a8ccc8-4e3b-4bfd-9fcf-0acb31e02ae9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.795107] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a23f434a-0779-4b43-9da6-dc8ad0abf215 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.832740] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9094299b-f888-4824-a0a9-2796a0d0ffa5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.842855] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c37abb2a-54f8-40e8-a4ee-c1c072cd65af {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.865028] env[62499]: DEBUG nova.compute.provider_tree [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 599.877136] env[62499]: ERROR nova.compute.manager [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 6eb83bc1-e5fd-4aba-bb63-fc8d979dbabf, please check neutron logs for more information. [ 599.877136] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 599.877136] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 599.877136] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 599.877136] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 599.877136] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 599.877136] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 599.877136] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 599.877136] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 599.877136] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 599.877136] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 599.877136] env[62499]: ERROR nova.compute.manager raise self.value [ 599.877136] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 599.877136] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 599.877136] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 599.877136] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 599.877978] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 599.877978] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 599.877978] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 6eb83bc1-e5fd-4aba-bb63-fc8d979dbabf, please check neutron logs for more information. [ 599.877978] env[62499]: ERROR nova.compute.manager [ 599.877978] env[62499]: Traceback (most recent call last): [ 599.877978] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 599.877978] env[62499]: listener.cb(fileno) [ 599.877978] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 599.877978] env[62499]: result = function(*args, **kwargs) [ 599.877978] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 599.877978] env[62499]: return func(*args, **kwargs) [ 599.877978] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 599.877978] env[62499]: raise e [ 599.877978] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 599.877978] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 599.877978] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 599.877978] env[62499]: created_port_ids = self._update_ports_for_instance( [ 599.877978] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 599.877978] env[62499]: with excutils.save_and_reraise_exception(): [ 599.877978] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 599.877978] env[62499]: self.force_reraise() [ 599.877978] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 599.877978] env[62499]: raise self.value [ 599.877978] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 599.877978] env[62499]: updated_port = self._update_port( [ 599.877978] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 599.877978] env[62499]: _ensure_no_port_binding_failure(port) [ 599.877978] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 599.877978] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 599.879556] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 6eb83bc1-e5fd-4aba-bb63-fc8d979dbabf, please check neutron logs for more information. [ 599.879556] env[62499]: Removing descriptor: 17 [ 599.879556] env[62499]: ERROR nova.compute.manager [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 6eb83bc1-e5fd-4aba-bb63-fc8d979dbabf, please check neutron logs for more information. [ 599.879556] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Traceback (most recent call last): [ 599.879556] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 599.879556] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] yield resources [ 599.879556] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 599.879556] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] self.driver.spawn(context, instance, image_meta, [ 599.879556] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 599.879556] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] self._vmops.spawn(context, instance, image_meta, injected_files, [ 599.879556] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 599.879556] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] vm_ref = self.build_virtual_machine(instance, [ 599.879959] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 599.879959] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] vif_infos = vmwarevif.get_vif_info(self._session, [ 599.879959] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 599.879959] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] for vif in network_info: [ 599.879959] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 599.879959] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] return self._sync_wrapper(fn, *args, **kwargs) [ 599.879959] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 599.879959] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] self.wait() [ 599.879959] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 599.879959] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] self[:] = self._gt.wait() [ 599.879959] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 599.879959] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] return self._exit_event.wait() [ 599.879959] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 599.881352] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] result = hub.switch() [ 599.881352] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 599.881352] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] return self.greenlet.switch() [ 599.881352] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 599.881352] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] result = function(*args, **kwargs) [ 599.881352] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 599.881352] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] return func(*args, **kwargs) [ 599.881352] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 599.881352] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] raise e [ 599.881352] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 599.881352] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] nwinfo = self.network_api.allocate_for_instance( [ 599.881352] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 599.881352] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] created_port_ids = self._update_ports_for_instance( [ 599.881771] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 599.881771] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] with excutils.save_and_reraise_exception(): [ 599.881771] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 599.881771] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] self.force_reraise() [ 599.881771] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 599.881771] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] raise self.value [ 599.881771] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 599.881771] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] updated_port = self._update_port( [ 599.881771] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 599.881771] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] _ensure_no_port_binding_failure(port) [ 599.881771] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 599.881771] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] raise exception.PortBindingFailed(port_id=port['id']) [ 599.882292] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] nova.exception.PortBindingFailed: Binding failed for port 6eb83bc1-e5fd-4aba-bb63-fc8d979dbabf, please check neutron logs for more information. [ 599.882292] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] [ 599.882292] env[62499]: INFO nova.compute.manager [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Terminating instance [ 599.882292] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Acquiring lock "refresh_cache-7661165d-2731-456e-af25-da13c68fd9fb" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 599.882425] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Acquired lock "refresh_cache-7661165d-2731-456e-af25-da13c68fd9fb" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 599.882581] env[62499]: DEBUG nova.network.neutron [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 599.998077] env[62499]: DEBUG nova.network.neutron [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 600.369348] env[62499]: DEBUG nova.scheduler.client.report [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 600.422702] env[62499]: DEBUG nova.network.neutron [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 600.501030] env[62499]: INFO nova.compute.manager [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] [instance: 76cf7f28-7413-4b06-bd30-ae22bbc2999e] Took 1.03 seconds to deallocate network for instance. [ 600.528925] env[62499]: DEBUG nova.network.neutron [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 600.614187] env[62499]: DEBUG oslo_concurrency.lockutils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Acquiring lock "0899249c-880c-48de-b2c3-2c407115463e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 600.614404] env[62499]: DEBUG oslo_concurrency.lockutils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Lock "0899249c-880c-48de-b2c3-2c407115463e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 600.693516] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Acquiring lock "a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 600.693783] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Lock "a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 600.876371] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.543s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 600.876935] env[62499]: DEBUG nova.compute.manager [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 600.880931] env[62499]: DEBUG oslo_concurrency.lockutils [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 17.994s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 600.881176] env[62499]: DEBUG nova.objects.instance [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Trying to apply a migration context that does not seem to be set for this instance {{(pid=62499) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 601.032325] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Releasing lock "refresh_cache-7661165d-2731-456e-af25-da13c68fd9fb" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 601.032460] env[62499]: DEBUG nova.compute.manager [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 601.032688] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 601.033024] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5a6f462d-0d99-448b-9036-66e6a2066a2a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.046385] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a19836b0-6bcf-475f-81c7-05040bc582d4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.073587] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7661165d-2731-456e-af25-da13c68fd9fb could not be found. [ 601.074507] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 601.074831] env[62499]: INFO nova.compute.manager [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Took 0.04 seconds to destroy the instance on the hypervisor. [ 601.075146] env[62499]: DEBUG oslo.service.loopingcall [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 601.075416] env[62499]: DEBUG nova.compute.manager [-] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 601.075556] env[62499]: DEBUG nova.network.neutron [-] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 601.099030] env[62499]: DEBUG nova.network.neutron [-] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 601.373698] env[62499]: DEBUG nova.compute.manager [req-c099f0c0-1e6a-4f2a-ba12-d8f86afe5ec8 req-07db36ca-dc7b-4f16-8a48-be83b7fc7b60 service nova] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Received event network-changed-6eb83bc1-e5fd-4aba-bb63-fc8d979dbabf {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 601.373996] env[62499]: DEBUG nova.compute.manager [req-c099f0c0-1e6a-4f2a-ba12-d8f86afe5ec8 req-07db36ca-dc7b-4f16-8a48-be83b7fc7b60 service nova] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Refreshing instance network info cache due to event network-changed-6eb83bc1-e5fd-4aba-bb63-fc8d979dbabf. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 601.374103] env[62499]: DEBUG oslo_concurrency.lockutils [req-c099f0c0-1e6a-4f2a-ba12-d8f86afe5ec8 req-07db36ca-dc7b-4f16-8a48-be83b7fc7b60 service nova] Acquiring lock "refresh_cache-7661165d-2731-456e-af25-da13c68fd9fb" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 601.374242] env[62499]: DEBUG oslo_concurrency.lockutils [req-c099f0c0-1e6a-4f2a-ba12-d8f86afe5ec8 req-07db36ca-dc7b-4f16-8a48-be83b7fc7b60 service nova] Acquired lock "refresh_cache-7661165d-2731-456e-af25-da13c68fd9fb" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 601.375284] env[62499]: DEBUG nova.network.neutron [req-c099f0c0-1e6a-4f2a-ba12-d8f86afe5ec8 req-07db36ca-dc7b-4f16-8a48-be83b7fc7b60 service nova] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Refreshing network info cache for port 6eb83bc1-e5fd-4aba-bb63-fc8d979dbabf {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 601.387978] env[62499]: DEBUG nova.compute.utils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 601.393756] env[62499]: DEBUG nova.compute.manager [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 601.393756] env[62499]: DEBUG nova.network.neutron [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 601.527257] env[62499]: DEBUG nova.policy [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a2ff9565717a4cceba792f68ffc03536', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7a5d82053bc44cb3a074fc125f58d068', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 601.548758] env[62499]: INFO nova.scheduler.client.report [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Deleted allocations for instance 76cf7f28-7413-4b06-bd30-ae22bbc2999e [ 601.607551] env[62499]: DEBUG nova.network.neutron [-] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 601.895160] env[62499]: DEBUG nova.compute.manager [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 601.901021] env[62499]: DEBUG oslo_concurrency.lockutils [None req-912f518f-55c0-40c5-b2ec-8b2214614842 tempest-ServersAdmin275Test-1881070433 tempest-ServersAdmin275Test-1881070433-project-admin] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.018s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 601.901021] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.652s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 601.908678] env[62499]: DEBUG nova.network.neutron [req-c099f0c0-1e6a-4f2a-ba12-d8f86afe5ec8 req-07db36ca-dc7b-4f16-8a48-be83b7fc7b60 service nova] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 601.996751] env[62499]: DEBUG nova.network.neutron [req-c099f0c0-1e6a-4f2a-ba12-d8f86afe5ec8 req-07db36ca-dc7b-4f16-8a48-be83b7fc7b60 service nova] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 602.062190] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2605a6f9-bb56-4a61-8489-4216b170e66b tempest-ServerExternalEventsTest-941014284 tempest-ServerExternalEventsTest-941014284-project-member] Lock "76cf7f28-7413-4b06-bd30-ae22bbc2999e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 50.515s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 602.111900] env[62499]: INFO nova.compute.manager [-] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Took 1.04 seconds to deallocate network for instance. [ 602.115750] env[62499]: DEBUG nova.compute.claims [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 602.116008] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 602.125467] env[62499]: DEBUG nova.network.neutron [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Successfully created port: 965c7a6a-7b9b-49b0-8f72-a6eea9ca2675 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 602.500653] env[62499]: DEBUG oslo_concurrency.lockutils [req-c099f0c0-1e6a-4f2a-ba12-d8f86afe5ec8 req-07db36ca-dc7b-4f16-8a48-be83b7fc7b60 service nova] Releasing lock "refresh_cache-7661165d-2731-456e-af25-da13c68fd9fb" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 602.500653] env[62499]: DEBUG nova.compute.manager [req-c099f0c0-1e6a-4f2a-ba12-d8f86afe5ec8 req-07db36ca-dc7b-4f16-8a48-be83b7fc7b60 service nova] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Received event network-vif-deleted-6eb83bc1-e5fd-4aba-bb63-fc8d979dbabf {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 602.566178] env[62499]: DEBUG nova.compute.manager [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 602.808019] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab1bc0a4-3c36-4164-b5f8-ebd4bd19d439 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.815930] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8fe20e4-74dd-4d66-9ad9-e77d914e9922 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.852210] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d237c5ca-6625-41a3-b6a8-aec877041a37 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.860899] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64f94a48-bb55-4a4c-8722-2dbb113f4f8c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.876571] env[62499]: DEBUG nova.compute.provider_tree [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 602.904665] env[62499]: DEBUG nova.compute.manager [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 602.941675] env[62499]: DEBUG nova.virt.hardware [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 602.941675] env[62499]: DEBUG nova.virt.hardware [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 602.941675] env[62499]: DEBUG nova.virt.hardware [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 602.941675] env[62499]: DEBUG nova.virt.hardware [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 602.942026] env[62499]: DEBUG nova.virt.hardware [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 602.942026] env[62499]: DEBUG nova.virt.hardware [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 602.942026] env[62499]: DEBUG nova.virt.hardware [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 602.942026] env[62499]: DEBUG nova.virt.hardware [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 602.942026] env[62499]: DEBUG nova.virt.hardware [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 602.942338] env[62499]: DEBUG nova.virt.hardware [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 602.942338] env[62499]: DEBUG nova.virt.hardware [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 602.942646] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-520f7ac4-0d2b-4b69-9b93-adc1e822d266 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.955640] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2f5ec3d-9529-4a88-8a47-985fa672abc5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.985108] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d57fe4e9-5fe7-4a93-a464-e4d5ae092655 tempest-ServersListShow296Test-1004933319 tempest-ServersListShow296Test-1004933319-project-member] Acquiring lock "da2b873c-0cac-4a7d-8639-b13e0f31b7b6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 602.985108] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d57fe4e9-5fe7-4a93-a464-e4d5ae092655 tempest-ServersListShow296Test-1004933319 tempest-ServersListShow296Test-1004933319-project-member] Lock "da2b873c-0cac-4a7d-8639-b13e0f31b7b6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 603.118943] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 603.380344] env[62499]: DEBUG nova.scheduler.client.report [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 603.443208] env[62499]: ERROR nova.compute.manager [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 965c7a6a-7b9b-49b0-8f72-a6eea9ca2675, please check neutron logs for more information. [ 603.443208] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 603.443208] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 603.443208] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 603.443208] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 603.443208] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 603.443208] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 603.443208] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 603.443208] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 603.443208] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 603.443208] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 603.443208] env[62499]: ERROR nova.compute.manager raise self.value [ 603.443208] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 603.443208] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 603.443208] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 603.443208] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 603.444219] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 603.444219] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 603.444219] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 965c7a6a-7b9b-49b0-8f72-a6eea9ca2675, please check neutron logs for more information. [ 603.444219] env[62499]: ERROR nova.compute.manager [ 603.444219] env[62499]: Traceback (most recent call last): [ 603.444219] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 603.444219] env[62499]: listener.cb(fileno) [ 603.444219] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 603.444219] env[62499]: result = function(*args, **kwargs) [ 603.444219] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 603.444219] env[62499]: return func(*args, **kwargs) [ 603.444219] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 603.444219] env[62499]: raise e [ 603.444219] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 603.444219] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 603.444219] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 603.444219] env[62499]: created_port_ids = self._update_ports_for_instance( [ 603.444219] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 603.444219] env[62499]: with excutils.save_and_reraise_exception(): [ 603.444219] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 603.444219] env[62499]: self.force_reraise() [ 603.444219] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 603.444219] env[62499]: raise self.value [ 603.444219] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 603.444219] env[62499]: updated_port = self._update_port( [ 603.444219] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 603.444219] env[62499]: _ensure_no_port_binding_failure(port) [ 603.444219] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 603.444219] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 603.445228] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 965c7a6a-7b9b-49b0-8f72-a6eea9ca2675, please check neutron logs for more information. [ 603.445228] env[62499]: Removing descriptor: 17 [ 603.445228] env[62499]: ERROR nova.compute.manager [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 965c7a6a-7b9b-49b0-8f72-a6eea9ca2675, please check neutron logs for more information. [ 603.445228] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Traceback (most recent call last): [ 603.445228] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 603.445228] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] yield resources [ 603.445228] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 603.445228] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] self.driver.spawn(context, instance, image_meta, [ 603.445228] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 603.445228] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 603.445228] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 603.445228] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] vm_ref = self.build_virtual_machine(instance, [ 603.445745] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 603.445745] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] vif_infos = vmwarevif.get_vif_info(self._session, [ 603.445745] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 603.445745] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] for vif in network_info: [ 603.445745] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 603.445745] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] return self._sync_wrapper(fn, *args, **kwargs) [ 603.445745] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 603.445745] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] self.wait() [ 603.445745] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 603.445745] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] self[:] = self._gt.wait() [ 603.445745] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 603.445745] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] return self._exit_event.wait() [ 603.445745] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 603.446218] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] result = hub.switch() [ 603.446218] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 603.446218] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] return self.greenlet.switch() [ 603.446218] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 603.446218] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] result = function(*args, **kwargs) [ 603.446218] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 603.446218] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] return func(*args, **kwargs) [ 603.446218] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 603.446218] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] raise e [ 603.446218] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 603.446218] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] nwinfo = self.network_api.allocate_for_instance( [ 603.446218] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 603.446218] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] created_port_ids = self._update_ports_for_instance( [ 603.448296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 603.448296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] with excutils.save_and_reraise_exception(): [ 603.448296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 603.448296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] self.force_reraise() [ 603.448296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 603.448296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] raise self.value [ 603.448296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 603.448296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] updated_port = self._update_port( [ 603.448296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 603.448296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] _ensure_no_port_binding_failure(port) [ 603.448296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 603.448296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] raise exception.PortBindingFailed(port_id=port['id']) [ 603.449011] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] nova.exception.PortBindingFailed: Binding failed for port 965c7a6a-7b9b-49b0-8f72-a6eea9ca2675, please check neutron logs for more information. [ 603.449011] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] [ 603.449011] env[62499]: INFO nova.compute.manager [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Terminating instance [ 603.449011] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Acquiring lock "refresh_cache-037217df-a3ed-4cef-9478-a9cc576a2f7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 603.449011] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Acquired lock "refresh_cache-037217df-a3ed-4cef-9478-a9cc576a2f7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 603.449011] env[62499]: DEBUG nova.network.neutron [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 603.682272] env[62499]: DEBUG oslo_concurrency.lockutils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquiring lock "3d9ac347-ffcc-4840-8090-811dbd2865ef" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 603.686021] env[62499]: DEBUG oslo_concurrency.lockutils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "3d9ac347-ffcc-4840-8090-811dbd2865ef" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 603.885594] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.986s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 603.886336] env[62499]: ERROR nova.compute.manager [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 149941b7-e36f-4036-b7de-c89202397eac, please check neutron logs for more information. [ 603.886336] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Traceback (most recent call last): [ 603.886336] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 603.886336] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] self.driver.spawn(context, instance, image_meta, [ 603.886336] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 603.886336] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 603.886336] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 603.886336] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] vm_ref = self.build_virtual_machine(instance, [ 603.886336] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 603.886336] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] vif_infos = vmwarevif.get_vif_info(self._session, [ 603.886336] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 603.886807] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] for vif in network_info: [ 603.886807] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 603.886807] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] return self._sync_wrapper(fn, *args, **kwargs) [ 603.886807] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 603.886807] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] self.wait() [ 603.886807] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 603.886807] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] self[:] = self._gt.wait() [ 603.886807] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 603.886807] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] return self._exit_event.wait() [ 603.886807] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 603.886807] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] result = hub.switch() [ 603.886807] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 603.886807] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] return self.greenlet.switch() [ 603.887258] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 603.887258] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] result = function(*args, **kwargs) [ 603.887258] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 603.887258] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] return func(*args, **kwargs) [ 603.887258] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 603.887258] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] raise e [ 603.887258] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 603.887258] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] nwinfo = self.network_api.allocate_for_instance( [ 603.887258] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 603.887258] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] created_port_ids = self._update_ports_for_instance( [ 603.887258] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 603.887258] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] with excutils.save_and_reraise_exception(): [ 603.887258] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 603.887663] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] self.force_reraise() [ 603.887663] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 603.887663] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] raise self.value [ 603.887663] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 603.887663] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] updated_port = self._update_port( [ 603.887663] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 603.887663] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] _ensure_no_port_binding_failure(port) [ 603.887663] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 603.887663] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] raise exception.PortBindingFailed(port_id=port['id']) [ 603.887663] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] nova.exception.PortBindingFailed: Binding failed for port 149941b7-e36f-4036-b7de-c89202397eac, please check neutron logs for more information. [ 603.887663] env[62499]: ERROR nova.compute.manager [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] [ 603.888073] env[62499]: DEBUG nova.compute.utils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Binding failed for port 149941b7-e36f-4036-b7de-c89202397eac, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 603.891941] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 18.015s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 603.893692] env[62499]: INFO nova.compute.claims [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 603.896428] env[62499]: DEBUG nova.compute.manager [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Build of instance 8d45497d-ba77-45b1-91f3-719b2689bee3 was re-scheduled: Binding failed for port 149941b7-e36f-4036-b7de-c89202397eac, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 603.897610] env[62499]: DEBUG nova.compute.manager [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 603.897858] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Acquiring lock "refresh_cache-8d45497d-ba77-45b1-91f3-719b2689bee3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 603.898028] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Acquired lock "refresh_cache-8d45497d-ba77-45b1-91f3-719b2689bee3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 603.898205] env[62499]: DEBUG nova.network.neutron [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 603.931989] env[62499]: DEBUG nova.compute.manager [req-5407f8dd-3982-40fe-82d7-dffcbff4dfb1 req-1b60c4a0-6da2-4f55-88bd-098531dab0c4 service nova] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Received event network-changed-965c7a6a-7b9b-49b0-8f72-a6eea9ca2675 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 603.931989] env[62499]: DEBUG nova.compute.manager [req-5407f8dd-3982-40fe-82d7-dffcbff4dfb1 req-1b60c4a0-6da2-4f55-88bd-098531dab0c4 service nova] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Refreshing instance network info cache due to event network-changed-965c7a6a-7b9b-49b0-8f72-a6eea9ca2675. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 603.931989] env[62499]: DEBUG oslo_concurrency.lockutils [req-5407f8dd-3982-40fe-82d7-dffcbff4dfb1 req-1b60c4a0-6da2-4f55-88bd-098531dab0c4 service nova] Acquiring lock "refresh_cache-037217df-a3ed-4cef-9478-a9cc576a2f7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 603.981271] env[62499]: DEBUG nova.network.neutron [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 604.088393] env[62499]: DEBUG nova.network.neutron [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 604.428123] env[62499]: DEBUG nova.network.neutron [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 604.594072] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Releasing lock "refresh_cache-037217df-a3ed-4cef-9478-a9cc576a2f7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 604.594791] env[62499]: DEBUG nova.compute.manager [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 604.595016] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 604.595365] env[62499]: DEBUG oslo_concurrency.lockutils [req-5407f8dd-3982-40fe-82d7-dffcbff4dfb1 req-1b60c4a0-6da2-4f55-88bd-098531dab0c4 service nova] Acquired lock "refresh_cache-037217df-a3ed-4cef-9478-a9cc576a2f7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 604.595537] env[62499]: DEBUG nova.network.neutron [req-5407f8dd-3982-40fe-82d7-dffcbff4dfb1 req-1b60c4a0-6da2-4f55-88bd-098531dab0c4 service nova] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Refreshing network info cache for port 965c7a6a-7b9b-49b0-8f72-a6eea9ca2675 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 604.596621] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1f127688-bcef-4a3b-b693-83899fbde157 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 604.609294] env[62499]: DEBUG nova.network.neutron [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 604.615365] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5d344b2-8812-41a6-8e8f-4d196c475ac9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 604.648023] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 037217df-a3ed-4cef-9478-a9cc576a2f7d could not be found. [ 604.648023] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 604.648023] env[62499]: INFO nova.compute.manager [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Took 0.05 seconds to destroy the instance on the hypervisor. [ 604.648300] env[62499]: DEBUG oslo.service.loopingcall [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 604.648606] env[62499]: DEBUG nova.compute.manager [-] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 604.648817] env[62499]: DEBUG nova.network.neutron [-] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 604.680537] env[62499]: DEBUG nova.network.neutron [-] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 605.113440] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Releasing lock "refresh_cache-8d45497d-ba77-45b1-91f3-719b2689bee3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 605.113743] env[62499]: DEBUG nova.compute.manager [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 605.113861] env[62499]: DEBUG nova.compute.manager [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 605.114034] env[62499]: DEBUG nova.network.neutron [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 605.126637] env[62499]: DEBUG nova.network.neutron [req-5407f8dd-3982-40fe-82d7-dffcbff4dfb1 req-1b60c4a0-6da2-4f55-88bd-098531dab0c4 service nova] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 605.142753] env[62499]: DEBUG nova.network.neutron [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 605.182717] env[62499]: DEBUG nova.network.neutron [-] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 605.294467] env[62499]: DEBUG nova.network.neutron [req-5407f8dd-3982-40fe-82d7-dffcbff4dfb1 req-1b60c4a0-6da2-4f55-88bd-098531dab0c4 service nova] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 605.345071] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1eb819a-2b19-44b1-8a36-91b537a2e5f7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.354517] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d9a41344-16dc-4260-a868-ed0979e16798 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.389886] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2b8488bd-637b-486a-bd42-49c8eafc096a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.398535] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab91c44b-a93d-46a4-97f7-7f1a9884fc7b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.415864] env[62499]: DEBUG nova.compute.provider_tree [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 605.647548] env[62499]: DEBUG nova.network.neutron [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 605.686209] env[62499]: INFO nova.compute.manager [-] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Took 1.04 seconds to deallocate network for instance. [ 605.688748] env[62499]: DEBUG nova.compute.claims [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 605.688955] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 605.797276] env[62499]: DEBUG oslo_concurrency.lockutils [req-5407f8dd-3982-40fe-82d7-dffcbff4dfb1 req-1b60c4a0-6da2-4f55-88bd-098531dab0c4 service nova] Releasing lock "refresh_cache-037217df-a3ed-4cef-9478-a9cc576a2f7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 605.919972] env[62499]: DEBUG nova.scheduler.client.report [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 606.129136] env[62499]: DEBUG nova.compute.manager [req-b7618fb4-ac60-458e-94a7-16fab20a7fef req-17741f57-a8f8-46ca-a616-2e008eff348a service nova] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Received event network-vif-deleted-965c7a6a-7b9b-49b0-8f72-a6eea9ca2675 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 606.150055] env[62499]: INFO nova.compute.manager [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] [instance: 8d45497d-ba77-45b1-91f3-719b2689bee3] Took 1.04 seconds to deallocate network for instance. [ 606.428526] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.536s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 606.429071] env[62499]: DEBUG nova.compute.manager [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 606.431955] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 17.960s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 606.432254] env[62499]: DEBUG nova.objects.instance [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Lazy-loading 'resources' on Instance uuid 27103ce2-0962-4fce-9331-d74179b0510c {{(pid=62499) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 606.669143] env[62499]: DEBUG oslo_concurrency.lockutils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Acquiring lock "3649431e-f856-4a28-9af7-8f101c3fc2d6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 606.669451] env[62499]: DEBUG oslo_concurrency.lockutils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Lock "3649431e-f856-4a28-9af7-8f101c3fc2d6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 606.938090] env[62499]: DEBUG nova.compute.utils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 606.938090] env[62499]: DEBUG nova.compute.manager [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 606.938090] env[62499]: DEBUG nova.network.neutron [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 606.987797] env[62499]: DEBUG nova.policy [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6ddf09ee6bce4ab3b468d1ecd2095992', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ce8145b6c7084b12b5acacb131b54534', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 607.201647] env[62499]: INFO nova.scheduler.client.report [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Deleted allocations for instance 8d45497d-ba77-45b1-91f3-719b2689bee3 [ 607.334503] env[62499]: DEBUG nova.network.neutron [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Successfully created port: d22afec3-7f0e-413f-b7ee-8529d26f4c99 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 607.374333] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f585bbd5-f9d7-4a8b-a596-091c759bfe2c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 607.390055] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-faef9d15-75fd-4bb4-8b37-fa67a1e91f80 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 607.427194] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7b97464c-5cb0-4f33-8292-d83a87f17a2a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 607.435651] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ad0528f-7c5a-41db-a8bf-e3f9110b44c5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 607.440268] env[62499]: DEBUG nova.compute.manager [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 607.450464] env[62499]: DEBUG nova.compute.provider_tree [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 607.711157] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c0934103-e766-4f78-851e-90f324c75734 tempest-ServersTestJSON-162904896 tempest-ServersTestJSON-162904896-project-member] Lock "8d45497d-ba77-45b1-91f3-719b2689bee3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 54.820s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 607.960127] env[62499]: DEBUG nova.scheduler.client.report [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 608.214701] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 608.370897] env[62499]: DEBUG nova.compute.manager [req-2d83635e-507f-49bf-a642-8352d1a1292a req-a0c1821b-9046-4317-9820-86f714275afc service nova] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Received event network-changed-d22afec3-7f0e-413f-b7ee-8529d26f4c99 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 608.370897] env[62499]: DEBUG nova.compute.manager [req-2d83635e-507f-49bf-a642-8352d1a1292a req-a0c1821b-9046-4317-9820-86f714275afc service nova] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Refreshing instance network info cache due to event network-changed-d22afec3-7f0e-413f-b7ee-8529d26f4c99. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 608.370897] env[62499]: DEBUG oslo_concurrency.lockutils [req-2d83635e-507f-49bf-a642-8352d1a1292a req-a0c1821b-9046-4317-9820-86f714275afc service nova] Acquiring lock "refresh_cache-9160ca3a-2aa2-4cfc-9900-3e4c770d2657" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 608.370897] env[62499]: DEBUG oslo_concurrency.lockutils [req-2d83635e-507f-49bf-a642-8352d1a1292a req-a0c1821b-9046-4317-9820-86f714275afc service nova] Acquired lock "refresh_cache-9160ca3a-2aa2-4cfc-9900-3e4c770d2657" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 608.370897] env[62499]: DEBUG nova.network.neutron [req-2d83635e-507f-49bf-a642-8352d1a1292a req-a0c1821b-9046-4317-9820-86f714275afc service nova] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Refreshing network info cache for port d22afec3-7f0e-413f-b7ee-8529d26f4c99 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 608.434643] env[62499]: ERROR nova.compute.manager [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d22afec3-7f0e-413f-b7ee-8529d26f4c99, please check neutron logs for more information. [ 608.434643] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 608.434643] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 608.434643] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 608.434643] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 608.434643] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 608.434643] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 608.434643] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 608.434643] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 608.434643] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 608.434643] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 608.434643] env[62499]: ERROR nova.compute.manager raise self.value [ 608.434643] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 608.434643] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 608.434643] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 608.434643] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 608.435795] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 608.435795] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 608.435795] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d22afec3-7f0e-413f-b7ee-8529d26f4c99, please check neutron logs for more information. [ 608.435795] env[62499]: ERROR nova.compute.manager [ 608.435795] env[62499]: Traceback (most recent call last): [ 608.435795] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 608.435795] env[62499]: listener.cb(fileno) [ 608.435795] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 608.435795] env[62499]: result = function(*args, **kwargs) [ 608.435795] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 608.435795] env[62499]: return func(*args, **kwargs) [ 608.435795] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 608.435795] env[62499]: raise e [ 608.435795] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 608.435795] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 608.435795] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 608.435795] env[62499]: created_port_ids = self._update_ports_for_instance( [ 608.435795] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 608.435795] env[62499]: with excutils.save_and_reraise_exception(): [ 608.435795] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 608.435795] env[62499]: self.force_reraise() [ 608.435795] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 608.435795] env[62499]: raise self.value [ 608.435795] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 608.435795] env[62499]: updated_port = self._update_port( [ 608.435795] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 608.435795] env[62499]: _ensure_no_port_binding_failure(port) [ 608.435795] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 608.435795] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 608.437085] env[62499]: nova.exception.PortBindingFailed: Binding failed for port d22afec3-7f0e-413f-b7ee-8529d26f4c99, please check neutron logs for more information. [ 608.437085] env[62499]: Removing descriptor: 17 [ 608.466412] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 2.034s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 608.470065] env[62499]: DEBUG nova.compute.manager [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 608.472432] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 19.316s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 608.473118] env[62499]: DEBUG nova.objects.instance [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Lazy-loading 'resources' on Instance uuid ddd212aa-9f4a-4b9d-aa60-cd4b64580f84 {{(pid=62499) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 608.498728] env[62499]: DEBUG nova.virt.hardware [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 608.499030] env[62499]: DEBUG nova.virt.hardware [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 608.499183] env[62499]: DEBUG nova.virt.hardware [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 608.499360] env[62499]: DEBUG nova.virt.hardware [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 608.499510] env[62499]: DEBUG nova.virt.hardware [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 608.499657] env[62499]: DEBUG nova.virt.hardware [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 608.499876] env[62499]: DEBUG nova.virt.hardware [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 608.500062] env[62499]: DEBUG nova.virt.hardware [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 608.500238] env[62499]: DEBUG nova.virt.hardware [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 608.500400] env[62499]: DEBUG nova.virt.hardware [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 608.500593] env[62499]: DEBUG nova.virt.hardware [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 608.501777] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97bd97d2-fa7e-426e-a7ac-484dea935565 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.505104] env[62499]: INFO nova.scheduler.client.report [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Deleted allocations for instance 27103ce2-0962-4fce-9331-d74179b0510c [ 608.512892] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f827412d-1443-4cb7-809c-a6d3c63ee5a0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.529025] env[62499]: ERROR nova.compute.manager [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d22afec3-7f0e-413f-b7ee-8529d26f4c99, please check neutron logs for more information. [ 608.529025] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Traceback (most recent call last): [ 608.529025] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 608.529025] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] yield resources [ 608.529025] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 608.529025] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] self.driver.spawn(context, instance, image_meta, [ 608.529025] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 608.529025] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] self._vmops.spawn(context, instance, image_meta, injected_files, [ 608.529025] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 608.529025] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] vm_ref = self.build_virtual_machine(instance, [ 608.529025] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 608.532566] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] vif_infos = vmwarevif.get_vif_info(self._session, [ 608.532566] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 608.532566] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] for vif in network_info: [ 608.532566] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 608.532566] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] return self._sync_wrapper(fn, *args, **kwargs) [ 608.532566] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 608.532566] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] self.wait() [ 608.532566] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 608.532566] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] self[:] = self._gt.wait() [ 608.532566] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 608.532566] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] return self._exit_event.wait() [ 608.532566] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 608.532566] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] current.throw(*self._exc) [ 608.533836] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 608.533836] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] result = function(*args, **kwargs) [ 608.533836] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 608.533836] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] return func(*args, **kwargs) [ 608.533836] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 608.533836] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] raise e [ 608.533836] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 608.533836] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] nwinfo = self.network_api.allocate_for_instance( [ 608.533836] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 608.533836] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] created_port_ids = self._update_ports_for_instance( [ 608.533836] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 608.533836] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] with excutils.save_and_reraise_exception(): [ 608.533836] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 608.534539] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] self.force_reraise() [ 608.534539] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 608.534539] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] raise self.value [ 608.534539] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 608.534539] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] updated_port = self._update_port( [ 608.534539] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 608.534539] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] _ensure_no_port_binding_failure(port) [ 608.534539] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 608.534539] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] raise exception.PortBindingFailed(port_id=port['id']) [ 608.534539] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] nova.exception.PortBindingFailed: Binding failed for port d22afec3-7f0e-413f-b7ee-8529d26f4c99, please check neutron logs for more information. [ 608.534539] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] [ 608.534539] env[62499]: INFO nova.compute.manager [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Terminating instance [ 608.535195] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Acquiring lock "refresh_cache-9160ca3a-2aa2-4cfc-9900-3e4c770d2657" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 608.736476] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 608.893161] env[62499]: DEBUG nova.network.neutron [req-2d83635e-507f-49bf-a642-8352d1a1292a req-a0c1821b-9046-4317-9820-86f714275afc service nova] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 609.019944] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9d5bd3bf-ad69-4bd7-a8c2-700a74435c66 tempest-ServerDiagnosticsV248Test-1944406010 tempest-ServerDiagnosticsV248Test-1944406010-project-member] Lock "27103ce2-0962-4fce-9331-d74179b0510c" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 24.507s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 609.031412] env[62499]: DEBUG nova.network.neutron [req-2d83635e-507f-49bf-a642-8352d1a1292a req-a0c1821b-9046-4317-9820-86f714275afc service nova] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 609.204834] env[62499]: DEBUG oslo_concurrency.lockutils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "71fa08bc-a5c7-459c-b65b-5dacd9537281" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 609.205636] env[62499]: DEBUG oslo_concurrency.lockutils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "71fa08bc-a5c7-459c-b65b-5dacd9537281" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 609.426050] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-909286cb-02f4-47df-b290-abac1fea6670 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 609.437169] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8228da1c-47af-4a04-bb88-6402c13c3403 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 609.474064] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-24f011b3-f62a-436e-897a-8b3a899bdbad {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 609.482707] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-927d228c-4860-40d0-b067-af8c9e1ec364 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 609.497431] env[62499]: DEBUG nova.compute.provider_tree [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 609.536128] env[62499]: DEBUG oslo_concurrency.lockutils [req-2d83635e-507f-49bf-a642-8352d1a1292a req-a0c1821b-9046-4317-9820-86f714275afc service nova] Releasing lock "refresh_cache-9160ca3a-2aa2-4cfc-9900-3e4c770d2657" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 609.536522] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Acquired lock "refresh_cache-9160ca3a-2aa2-4cfc-9900-3e4c770d2657" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 609.536796] env[62499]: DEBUG nova.network.neutron [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 609.688344] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Acquiring lock "6ae11b59-68af-482a-ad65-543a56d7aaa4" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 609.692017] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Lock "6ae11b59-68af-482a-ad65-543a56d7aaa4" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 610.000893] env[62499]: DEBUG nova.scheduler.client.report [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 610.058760] env[62499]: DEBUG nova.network.neutron [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 610.233279] env[62499]: DEBUG nova.network.neutron [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 610.393727] env[62499]: DEBUG nova.compute.manager [req-55398a17-36ef-4bd0-88c4-0206486a5303 req-7a2b1aeb-31d8-4e0e-87d9-9cbd47217867 service nova] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Received event network-vif-deleted-d22afec3-7f0e-413f-b7ee-8529d26f4c99 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 610.507786] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 2.035s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 610.513052] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 20.222s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 610.540979] env[62499]: INFO nova.scheduler.client.report [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Deleted allocations for instance ddd212aa-9f4a-4b9d-aa60-cd4b64580f84 [ 610.739016] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Releasing lock "refresh_cache-9160ca3a-2aa2-4cfc-9900-3e4c770d2657" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 610.739016] env[62499]: DEBUG nova.compute.manager [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 610.739016] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 610.739016] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e7004f50-5fd5-4243-b899-fd0b5aa58a39 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 610.748328] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dc6edd5a-68a6-41b6-9e40-2790205ccd35 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 610.779717] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9160ca3a-2aa2-4cfc-9900-3e4c770d2657 could not be found. [ 610.780149] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 610.780526] env[62499]: INFO nova.compute.manager [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Took 0.04 seconds to destroy the instance on the hypervisor. [ 610.780903] env[62499]: DEBUG oslo.service.loopingcall [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 610.781268] env[62499]: DEBUG nova.compute.manager [-] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 610.781486] env[62499]: DEBUG nova.network.neutron [-] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 610.814055] env[62499]: DEBUG nova.network.neutron [-] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 611.053167] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1dbdf853-ae0b-411d-9792-8358f3a9169a tempest-ServersAdmin275Test-1946928354 tempest-ServersAdmin275Test-1946928354-project-member] Lock "ddd212aa-9f4a-4b9d-aa60-cd4b64580f84" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 25.954s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 611.316979] env[62499]: DEBUG nova.network.neutron [-] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 611.464148] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09166680-1b2a-4ca0-8cec-a684b0a651fb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 611.474827] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-efd16082-0288-420e-acb2-8980c7c306b2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 611.510840] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af2d6b75-5463-42da-be43-e6052f06bbd6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 611.523948] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec55f460-dcfa-4077-9611-f89240565bca {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 611.540868] env[62499]: DEBUG nova.compute.provider_tree [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 611.822926] env[62499]: INFO nova.compute.manager [-] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Took 1.04 seconds to deallocate network for instance. [ 611.824898] env[62499]: DEBUG nova.compute.claims [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 611.824898] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 612.044410] env[62499]: DEBUG nova.scheduler.client.report [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 612.550997] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.038s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 612.551387] env[62499]: ERROR nova.compute.manager [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1bf2c3a4-bfd7-4a69-a486-3f07a007056e, please check neutron logs for more information. [ 612.551387] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Traceback (most recent call last): [ 612.551387] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 612.551387] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] self.driver.spawn(context, instance, image_meta, [ 612.551387] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 612.551387] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 612.551387] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 612.551387] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] vm_ref = self.build_virtual_machine(instance, [ 612.551387] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 612.551387] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] vif_infos = vmwarevif.get_vif_info(self._session, [ 612.551387] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 612.551687] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] for vif in network_info: [ 612.551687] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 612.551687] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] return self._sync_wrapper(fn, *args, **kwargs) [ 612.551687] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 612.551687] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] self.wait() [ 612.551687] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 612.551687] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] self[:] = self._gt.wait() [ 612.551687] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 612.551687] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] return self._exit_event.wait() [ 612.551687] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 612.551687] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] result = hub.switch() [ 612.551687] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 612.551687] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] return self.greenlet.switch() [ 612.552081] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 612.552081] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] result = function(*args, **kwargs) [ 612.552081] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 612.552081] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] return func(*args, **kwargs) [ 612.552081] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 612.552081] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] raise e [ 612.552081] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 612.552081] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] nwinfo = self.network_api.allocate_for_instance( [ 612.552081] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 612.552081] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] created_port_ids = self._update_ports_for_instance( [ 612.552081] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 612.552081] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] with excutils.save_and_reraise_exception(): [ 612.552081] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 612.552365] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] self.force_reraise() [ 612.552365] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 612.552365] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] raise self.value [ 612.552365] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 612.552365] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] updated_port = self._update_port( [ 612.552365] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 612.552365] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] _ensure_no_port_binding_failure(port) [ 612.552365] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 612.552365] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] raise exception.PortBindingFailed(port_id=port['id']) [ 612.552365] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] nova.exception.PortBindingFailed: Binding failed for port 1bf2c3a4-bfd7-4a69-a486-3f07a007056e, please check neutron logs for more information. [ 612.552365] env[62499]: ERROR nova.compute.manager [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] [ 612.552608] env[62499]: DEBUG nova.compute.utils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Binding failed for port 1bf2c3a4-bfd7-4a69-a486-3f07a007056e, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 612.554066] env[62499]: DEBUG oslo_concurrency.lockutils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 19.864s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 612.557076] env[62499]: DEBUG nova.compute.manager [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Build of instance 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a was re-scheduled: Binding failed for port 1bf2c3a4-bfd7-4a69-a486-3f07a007056e, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 612.557216] env[62499]: DEBUG nova.compute.manager [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 612.557320] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Acquiring lock "refresh_cache-7c5b9fab-bb90-4c70-b445-fe0b637dbd4a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 612.557458] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Acquired lock "refresh_cache-7c5b9fab-bb90-4c70-b445-fe0b637dbd4a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 612.557612] env[62499]: DEBUG nova.network.neutron [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 613.095571] env[62499]: DEBUG nova.network.neutron [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 613.401747] env[62499]: DEBUG nova.network.neutron [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 613.480388] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e90b4f25-490d-4215-a632-797f66946272 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 613.489750] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e2092e4-a504-4315-a958-7a8a74238851 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 613.521363] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f871ed3-2720-4737-8710-b13d57d171e8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 613.530340] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72453f82-9224-4738-9082-3e771381a9b5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 613.544781] env[62499]: DEBUG nova.compute.provider_tree [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 613.902923] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Releasing lock "refresh_cache-7c5b9fab-bb90-4c70-b445-fe0b637dbd4a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 613.903193] env[62499]: DEBUG nova.compute.manager [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 613.903379] env[62499]: DEBUG nova.compute.manager [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 613.903551] env[62499]: DEBUG nova.network.neutron [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 613.926271] env[62499]: DEBUG nova.network.neutron [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 614.048083] env[62499]: DEBUG nova.scheduler.client.report [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 614.430009] env[62499]: DEBUG nova.network.neutron [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 614.553337] env[62499]: DEBUG oslo_concurrency.lockutils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 614.554246] env[62499]: ERROR nova.compute.manager [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3ecff7db-5412-4e0b-bc53-9a1924aa0f90, please check neutron logs for more information. [ 614.554246] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Traceback (most recent call last): [ 614.554246] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 614.554246] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] self.driver.spawn(context, instance, image_meta, [ 614.554246] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 614.554246] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] self._vmops.spawn(context, instance, image_meta, injected_files, [ 614.554246] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 614.554246] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] vm_ref = self.build_virtual_machine(instance, [ 614.554246] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 614.554246] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] vif_infos = vmwarevif.get_vif_info(self._session, [ 614.554246] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 614.554544] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] for vif in network_info: [ 614.554544] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 614.554544] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] return self._sync_wrapper(fn, *args, **kwargs) [ 614.554544] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 614.554544] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] self.wait() [ 614.554544] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 614.554544] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] self[:] = self._gt.wait() [ 614.554544] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 614.554544] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] return self._exit_event.wait() [ 614.554544] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 614.554544] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] result = hub.switch() [ 614.554544] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 614.554544] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] return self.greenlet.switch() [ 614.554842] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 614.554842] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] result = function(*args, **kwargs) [ 614.554842] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 614.554842] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] return func(*args, **kwargs) [ 614.554842] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 614.554842] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] raise e [ 614.554842] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 614.554842] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] nwinfo = self.network_api.allocate_for_instance( [ 614.554842] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 614.554842] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] created_port_ids = self._update_ports_for_instance( [ 614.554842] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 614.554842] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] with excutils.save_and_reraise_exception(): [ 614.554842] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 614.555113] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] self.force_reraise() [ 614.555113] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 614.555113] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] raise self.value [ 614.555113] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 614.555113] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] updated_port = self._update_port( [ 614.555113] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 614.555113] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] _ensure_no_port_binding_failure(port) [ 614.555113] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 614.555113] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] raise exception.PortBindingFailed(port_id=port['id']) [ 614.555113] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] nova.exception.PortBindingFailed: Binding failed for port 3ecff7db-5412-4e0b-bc53-9a1924aa0f90, please check neutron logs for more information. [ 614.555113] env[62499]: ERROR nova.compute.manager [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] [ 614.555350] env[62499]: DEBUG nova.compute.utils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Binding failed for port 3ecff7db-5412-4e0b-bc53-9a1924aa0f90, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 614.556123] env[62499]: DEBUG oslo_concurrency.lockutils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 20.563s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 614.560309] env[62499]: DEBUG nova.compute.manager [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Build of instance 3706243f-4768-4b44-9a5b-730bf2b18974 was re-scheduled: Binding failed for port 3ecff7db-5412-4e0b-bc53-9a1924aa0f90, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 614.560419] env[62499]: DEBUG nova.compute.manager [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 614.560608] env[62499]: DEBUG oslo_concurrency.lockutils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Acquiring lock "refresh_cache-3706243f-4768-4b44-9a5b-730bf2b18974" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 614.561425] env[62499]: DEBUG oslo_concurrency.lockutils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Acquired lock "refresh_cache-3706243f-4768-4b44-9a5b-730bf2b18974" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 614.561425] env[62499]: DEBUG nova.network.neutron [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 614.935509] env[62499]: INFO nova.compute.manager [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] [instance: 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a] Took 1.03 seconds to deallocate network for instance. [ 615.089652] env[62499]: DEBUG nova.network.neutron [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 615.261526] env[62499]: DEBUG nova.network.neutron [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 615.499537] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-946052b8-5e8e-467e-926e-00ba40dc1283 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 615.508464] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8bebfedc-7ad8-41e4-a63d-25aa8f14ec31 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 615.543072] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4628f073-73d4-44f0-bee3-6e80d2b8a8bc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 615.551955] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d6deb43-cfe3-4e2b-be6e-a0f8facd05dd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 615.568686] env[62499]: DEBUG nova.compute.provider_tree [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 615.763810] env[62499]: DEBUG oslo_concurrency.lockutils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Releasing lock "refresh_cache-3706243f-4768-4b44-9a5b-730bf2b18974" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 615.764355] env[62499]: DEBUG nova.compute.manager [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 615.764355] env[62499]: DEBUG nova.compute.manager [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 615.764519] env[62499]: DEBUG nova.network.neutron [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 615.790769] env[62499]: DEBUG nova.network.neutron [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 615.981928] env[62499]: INFO nova.scheduler.client.report [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Deleted allocations for instance 7c5b9fab-bb90-4c70-b445-fe0b637dbd4a [ 616.071714] env[62499]: DEBUG nova.scheduler.client.report [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 616.297764] env[62499]: DEBUG nova.network.neutron [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 616.493600] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0a08764-f186-403c-9a54-545a14e71922 tempest-ServersWithSpecificFlavorTestJSON-240076549 tempest-ServersWithSpecificFlavorTestJSON-240076549-project-member] Lock "7c5b9fab-bb90-4c70-b445-fe0b637dbd4a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 62.987s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 616.577206] env[62499]: DEBUG oslo_concurrency.lockutils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.021s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 616.577713] env[62499]: ERROR nova.compute.manager [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b5877af8-44ec-41fc-bf83-41d3e7cf35f6, please check neutron logs for more information. [ 616.577713] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Traceback (most recent call last): [ 616.577713] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 616.577713] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] self.driver.spawn(context, instance, image_meta, [ 616.577713] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 616.577713] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 616.577713] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 616.577713] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] vm_ref = self.build_virtual_machine(instance, [ 616.577713] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 616.577713] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] vif_infos = vmwarevif.get_vif_info(self._session, [ 616.577713] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 616.578163] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] for vif in network_info: [ 616.578163] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 616.578163] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] return self._sync_wrapper(fn, *args, **kwargs) [ 616.578163] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 616.578163] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] self.wait() [ 616.578163] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 616.578163] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] self[:] = self._gt.wait() [ 616.578163] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 616.578163] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] return self._exit_event.wait() [ 616.578163] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 616.578163] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] result = hub.switch() [ 616.578163] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 616.578163] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] return self.greenlet.switch() [ 616.578510] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 616.578510] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] result = function(*args, **kwargs) [ 616.578510] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 616.578510] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] return func(*args, **kwargs) [ 616.578510] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 616.578510] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] raise e [ 616.578510] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 616.578510] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] nwinfo = self.network_api.allocate_for_instance( [ 616.578510] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 616.578510] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] created_port_ids = self._update_ports_for_instance( [ 616.578510] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 616.578510] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] with excutils.save_and_reraise_exception(): [ 616.578510] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 616.578836] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] self.force_reraise() [ 616.578836] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 616.578836] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] raise self.value [ 616.578836] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 616.578836] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] updated_port = self._update_port( [ 616.578836] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 616.578836] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] _ensure_no_port_binding_failure(port) [ 616.578836] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 616.578836] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] raise exception.PortBindingFailed(port_id=port['id']) [ 616.578836] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] nova.exception.PortBindingFailed: Binding failed for port b5877af8-44ec-41fc-bf83-41d3e7cf35f6, please check neutron logs for more information. [ 616.578836] env[62499]: ERROR nova.compute.manager [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] [ 616.579207] env[62499]: DEBUG nova.compute.utils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Binding failed for port b5877af8-44ec-41fc-bf83-41d3e7cf35f6, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 616.580845] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.969s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 616.583848] env[62499]: INFO nova.compute.claims [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 616.590644] env[62499]: DEBUG nova.compute.manager [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Build of instance 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c was re-scheduled: Binding failed for port b5877af8-44ec-41fc-bf83-41d3e7cf35f6, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 616.594039] env[62499]: DEBUG nova.compute.manager [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 616.594039] env[62499]: DEBUG oslo_concurrency.lockutils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "refresh_cache-9c7433a1-9d5b-4ef2-ad65-f1217f0e862c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 616.594039] env[62499]: DEBUG oslo_concurrency.lockutils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquired lock "refresh_cache-9c7433a1-9d5b-4ef2-ad65-f1217f0e862c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 616.594039] env[62499]: DEBUG nova.network.neutron [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 616.731102] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Acquiring lock "6d8b7855-567c-44c3-90ee-fc0daec87f46" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 616.731471] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Lock "6d8b7855-567c-44c3-90ee-fc0daec87f46" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 616.802202] env[62499]: INFO nova.compute.manager [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] [instance: 3706243f-4768-4b44-9a5b-730bf2b18974] Took 1.04 seconds to deallocate network for instance. [ 616.996631] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 617.118064] env[62499]: DEBUG nova.network.neutron [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 617.275774] env[62499]: DEBUG nova.network.neutron [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 617.525691] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 617.781440] env[62499]: DEBUG oslo_concurrency.lockutils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Releasing lock "refresh_cache-9c7433a1-9d5b-4ef2-ad65-f1217f0e862c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 617.782084] env[62499]: DEBUG nova.compute.manager [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 617.782317] env[62499]: DEBUG nova.compute.manager [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 617.782594] env[62499]: DEBUG nova.network.neutron [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 617.807691] env[62499]: DEBUG nova.network.neutron [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 617.837526] env[62499]: INFO nova.scheduler.client.report [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Deleted allocations for instance 3706243f-4768-4b44-9a5b-730bf2b18974 [ 618.054012] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-41e65a8c-7fdd-4874-b5f9-c395470901ae {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 618.063232] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-10ea5b01-e8e4-42d7-9039-0d6ad6b19616 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 618.097863] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-43629b91-42d6-4977-92bf-9790c6f70c67 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 618.106855] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac2bb90d-cf48-4a8b-9b4a-2e00bc166d17 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 618.124713] env[62499]: DEBUG nova.compute.provider_tree [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 618.310643] env[62499]: DEBUG nova.network.neutron [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 618.345571] env[62499]: DEBUG oslo_concurrency.lockutils [None req-36db2140-89f2-4fd4-bb0d-71db22b3a4be tempest-VolumesAssistedSnapshotsTest-1771293483 tempest-VolumesAssistedSnapshotsTest-1771293483-project-member] Lock "3706243f-4768-4b44-9a5b-730bf2b18974" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 64.722s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 618.633544] env[62499]: DEBUG nova.scheduler.client.report [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 618.815712] env[62499]: INFO nova.compute.manager [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c] Took 1.03 seconds to deallocate network for instance. [ 618.848710] env[62499]: DEBUG nova.compute.manager [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 619.146019] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.562s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 619.146019] env[62499]: DEBUG nova.compute.manager [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 619.147024] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 19.388s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 619.277108] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Acquiring lock "8a769120-906c-4e7c-ac19-b7895039efc1" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 619.278671] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Lock "8a769120-906c-4e7c-ac19-b7895039efc1" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.002s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 619.374149] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 619.652132] env[62499]: DEBUG nova.compute.utils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 619.660022] env[62499]: DEBUG nova.compute.manager [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 619.660022] env[62499]: DEBUG nova.network.neutron [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 619.741777] env[62499]: DEBUG nova.policy [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f7f6adeea8054eefaf9d72e4264474c2', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd6e3f664f5d3403289b586dcceb28424', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 619.855800] env[62499]: INFO nova.scheduler.client.report [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Deleted allocations for instance 9c7433a1-9d5b-4ef2-ad65-f1217f0e862c [ 620.105340] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92a370e3-b7f0-43aa-b7b3-ec1aa6590fd5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 620.114509] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b81fee79-8986-486e-aebf-47938e279230 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 620.149294] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7e43ffb-ef1c-4040-8ede-fc5498a00db1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 620.157948] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-30ec79e3-83bd-4ad6-a842-a2fdde84a2d8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 620.162455] env[62499]: DEBUG nova.compute.manager [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 620.176629] env[62499]: DEBUG nova.compute.provider_tree [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 620.379273] env[62499]: DEBUG oslo_concurrency.lockutils [None req-03d5a889-1d0a-4746-9844-58ca11408bd2 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "9c7433a1-9d5b-4ef2-ad65-f1217f0e862c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 64.464s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 620.560738] env[62499]: DEBUG nova.network.neutron [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Successfully created port: c853a44f-5ddf-4d9b-b501-5cf4d5772cd4 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 620.679737] env[62499]: DEBUG nova.scheduler.client.report [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 620.886941] env[62499]: DEBUG nova.compute.manager [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 621.174123] env[62499]: DEBUG nova.compute.manager [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 621.188700] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.041s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 621.189365] env[62499]: ERROR nova.compute.manager [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 10bc8270-f0af-4ac1-984d-dedf660a797d, please check neutron logs for more information. [ 621.189365] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Traceback (most recent call last): [ 621.189365] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 621.189365] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] self.driver.spawn(context, instance, image_meta, [ 621.189365] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 621.189365] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] self._vmops.spawn(context, instance, image_meta, injected_files, [ 621.189365] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 621.189365] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] vm_ref = self.build_virtual_machine(instance, [ 621.189365] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 621.189365] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] vif_infos = vmwarevif.get_vif_info(self._session, [ 621.189365] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 621.189650] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] for vif in network_info: [ 621.189650] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 621.189650] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] return self._sync_wrapper(fn, *args, **kwargs) [ 621.189650] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 621.189650] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] self.wait() [ 621.189650] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 621.189650] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] self[:] = self._gt.wait() [ 621.189650] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 621.189650] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] return self._exit_event.wait() [ 621.189650] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 621.189650] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] result = hub.switch() [ 621.189650] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 621.189650] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] return self.greenlet.switch() [ 621.189924] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 621.189924] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] result = function(*args, **kwargs) [ 621.189924] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 621.189924] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] return func(*args, **kwargs) [ 621.189924] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 621.189924] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] raise e [ 621.189924] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 621.189924] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] nwinfo = self.network_api.allocate_for_instance( [ 621.189924] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 621.189924] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] created_port_ids = self._update_ports_for_instance( [ 621.189924] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 621.189924] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] with excutils.save_and_reraise_exception(): [ 621.189924] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 621.190200] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] self.force_reraise() [ 621.190200] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 621.190200] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] raise self.value [ 621.190200] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 621.190200] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] updated_port = self._update_port( [ 621.190200] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 621.190200] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] _ensure_no_port_binding_failure(port) [ 621.190200] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 621.190200] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] raise exception.PortBindingFailed(port_id=port['id']) [ 621.190200] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] nova.exception.PortBindingFailed: Binding failed for port 10bc8270-f0af-4ac1-984d-dedf660a797d, please check neutron logs for more information. [ 621.190200] env[62499]: ERROR nova.compute.manager [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] [ 621.191820] env[62499]: DEBUG nova.compute.utils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Binding failed for port 10bc8270-f0af-4ac1-984d-dedf660a797d, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 621.194574] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 19.078s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 621.200105] env[62499]: DEBUG nova.compute.manager [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Build of instance 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733 was re-scheduled: Binding failed for port 10bc8270-f0af-4ac1-984d-dedf660a797d, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 621.200592] env[62499]: DEBUG nova.compute.manager [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 621.200903] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Acquiring lock "refresh_cache-0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 621.201135] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Acquired lock "refresh_cache-0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 621.201251] env[62499]: DEBUG nova.network.neutron [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 621.213526] env[62499]: DEBUG nova.virt.hardware [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 621.213817] env[62499]: DEBUG nova.virt.hardware [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 621.214041] env[62499]: DEBUG nova.virt.hardware [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 621.215359] env[62499]: DEBUG nova.virt.hardware [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 621.215359] env[62499]: DEBUG nova.virt.hardware [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 621.215359] env[62499]: DEBUG nova.virt.hardware [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 621.215359] env[62499]: DEBUG nova.virt.hardware [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 621.215359] env[62499]: DEBUG nova.virt.hardware [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 621.215561] env[62499]: DEBUG nova.virt.hardware [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 621.215561] env[62499]: DEBUG nova.virt.hardware [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 621.216737] env[62499]: DEBUG nova.virt.hardware [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 621.217521] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb75d6e8-da07-40f4-a1d8-83da1cb53d52 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 621.228868] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bab776ef-f3b8-49a6-8dae-99303a0ad0cd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 621.415997] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 621.738602] env[62499]: DEBUG nova.network.neutron [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 621.899911] env[62499]: ERROR nova.compute.manager [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port c853a44f-5ddf-4d9b-b501-5cf4d5772cd4, please check neutron logs for more information. [ 621.899911] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 621.899911] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 621.899911] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 621.899911] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 621.899911] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 621.899911] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 621.899911] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 621.899911] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 621.899911] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 621.899911] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 621.899911] env[62499]: ERROR nova.compute.manager raise self.value [ 621.899911] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 621.899911] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 621.899911] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 621.899911] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 621.900321] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 621.900321] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 621.900321] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port c853a44f-5ddf-4d9b-b501-5cf4d5772cd4, please check neutron logs for more information. [ 621.900321] env[62499]: ERROR nova.compute.manager [ 621.900321] env[62499]: Traceback (most recent call last): [ 621.900321] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 621.900321] env[62499]: listener.cb(fileno) [ 621.900321] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 621.900321] env[62499]: result = function(*args, **kwargs) [ 621.900321] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 621.900321] env[62499]: return func(*args, **kwargs) [ 621.900321] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 621.900321] env[62499]: raise e [ 621.900321] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 621.900321] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 621.900321] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 621.900321] env[62499]: created_port_ids = self._update_ports_for_instance( [ 621.900321] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 621.900321] env[62499]: with excutils.save_and_reraise_exception(): [ 621.900321] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 621.900321] env[62499]: self.force_reraise() [ 621.900321] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 621.900321] env[62499]: raise self.value [ 621.900321] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 621.900321] env[62499]: updated_port = self._update_port( [ 621.900321] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 621.900321] env[62499]: _ensure_no_port_binding_failure(port) [ 621.900321] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 621.900321] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 621.900948] env[62499]: nova.exception.PortBindingFailed: Binding failed for port c853a44f-5ddf-4d9b-b501-5cf4d5772cd4, please check neutron logs for more information. [ 621.900948] env[62499]: Removing descriptor: 17 [ 621.900948] env[62499]: ERROR nova.compute.manager [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port c853a44f-5ddf-4d9b-b501-5cf4d5772cd4, please check neutron logs for more information. [ 621.900948] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Traceback (most recent call last): [ 621.900948] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 621.900948] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] yield resources [ 621.900948] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 621.900948] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] self.driver.spawn(context, instance, image_meta, [ 621.900948] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 621.900948] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 621.900948] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 621.900948] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] vm_ref = self.build_virtual_machine(instance, [ 621.901216] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 621.901216] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] vif_infos = vmwarevif.get_vif_info(self._session, [ 621.901216] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 621.901216] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] for vif in network_info: [ 621.901216] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 621.901216] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] return self._sync_wrapper(fn, *args, **kwargs) [ 621.901216] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 621.901216] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] self.wait() [ 621.901216] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 621.901216] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] self[:] = self._gt.wait() [ 621.901216] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 621.901216] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] return self._exit_event.wait() [ 621.901216] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 621.901516] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] result = hub.switch() [ 621.901516] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 621.901516] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] return self.greenlet.switch() [ 621.901516] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 621.901516] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] result = function(*args, **kwargs) [ 621.901516] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 621.901516] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] return func(*args, **kwargs) [ 621.901516] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 621.901516] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] raise e [ 621.901516] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 621.901516] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] nwinfo = self.network_api.allocate_for_instance( [ 621.901516] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 621.901516] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] created_port_ids = self._update_ports_for_instance( [ 621.901790] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 621.901790] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] with excutils.save_and_reraise_exception(): [ 621.901790] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 621.901790] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] self.force_reraise() [ 621.901790] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 621.901790] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] raise self.value [ 621.901790] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 621.901790] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] updated_port = self._update_port( [ 621.901790] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 621.901790] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] _ensure_no_port_binding_failure(port) [ 621.901790] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 621.901790] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] raise exception.PortBindingFailed(port_id=port['id']) [ 621.902066] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] nova.exception.PortBindingFailed: Binding failed for port c853a44f-5ddf-4d9b-b501-5cf4d5772cd4, please check neutron logs for more information. [ 621.902066] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] [ 621.902066] env[62499]: INFO nova.compute.manager [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Terminating instance [ 621.903347] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Acquiring lock "refresh_cache-81b1d279-dfad-4a2c-bcfb-043b13d77af6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 621.903347] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Acquired lock "refresh_cache-81b1d279-dfad-4a2c-bcfb-043b13d77af6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 621.905254] env[62499]: DEBUG nova.network.neutron [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 621.962574] env[62499]: DEBUG nova.network.neutron [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 622.020655] env[62499]: DEBUG nova.compute.manager [req-29530884-fa26-4501-a0ea-5d06e1896d9b req-6632173c-6ed4-4ba8-9940-0304d7a3defd service nova] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Received event network-changed-c853a44f-5ddf-4d9b-b501-5cf4d5772cd4 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 622.020857] env[62499]: DEBUG nova.compute.manager [req-29530884-fa26-4501-a0ea-5d06e1896d9b req-6632173c-6ed4-4ba8-9940-0304d7a3defd service nova] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Refreshing instance network info cache due to event network-changed-c853a44f-5ddf-4d9b-b501-5cf4d5772cd4. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 622.021060] env[62499]: DEBUG oslo_concurrency.lockutils [req-29530884-fa26-4501-a0ea-5d06e1896d9b req-6632173c-6ed4-4ba8-9940-0304d7a3defd service nova] Acquiring lock "refresh_cache-81b1d279-dfad-4a2c-bcfb-043b13d77af6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 622.124037] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39c9f40c-28ee-471b-804e-6054facc0fd8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.134539] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e8f3ea0f-046d-45b5-b1bd-583199b15a21 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.178584] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29083730-18e1-4e64-a656-18b1011d9d36 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.187354] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20ce516c-6f8a-45ce-9d31-bac4f47449c8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.202491] env[62499]: DEBUG nova.compute.provider_tree [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 622.429100] env[62499]: DEBUG nova.network.neutron [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 622.469686] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Releasing lock "refresh_cache-0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 622.469926] env[62499]: DEBUG nova.compute.manager [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 622.470113] env[62499]: DEBUG nova.compute.manager [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 622.470280] env[62499]: DEBUG nova.network.neutron [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 622.490305] env[62499]: DEBUG nova.network.neutron [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 622.601122] env[62499]: DEBUG nova.network.neutron [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 622.710399] env[62499]: DEBUG nova.scheduler.client.report [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 622.839877] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "75b04055-8c7e-4659-9451-ddcd6d39fe2e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 622.840118] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "75b04055-8c7e-4659-9451-ddcd6d39fe2e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.993956] env[62499]: DEBUG nova.network.neutron [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 623.103821] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Releasing lock "refresh_cache-81b1d279-dfad-4a2c-bcfb-043b13d77af6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 623.105068] env[62499]: DEBUG nova.compute.manager [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 623.105068] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 623.105068] env[62499]: DEBUG oslo_concurrency.lockutils [req-29530884-fa26-4501-a0ea-5d06e1896d9b req-6632173c-6ed4-4ba8-9940-0304d7a3defd service nova] Acquired lock "refresh_cache-81b1d279-dfad-4a2c-bcfb-043b13d77af6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 623.105068] env[62499]: DEBUG nova.network.neutron [req-29530884-fa26-4501-a0ea-5d06e1896d9b req-6632173c-6ed4-4ba8-9940-0304d7a3defd service nova] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Refreshing network info cache for port c853a44f-5ddf-4d9b-b501-5cf4d5772cd4 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 623.105922] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d6b192ea-cec1-4433-a8f7-dd74f6240e4a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 623.120180] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3668eead-64ae-49a8-906d-e77efb31466b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 623.145128] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 81b1d279-dfad-4a2c-bcfb-043b13d77af6 could not be found. [ 623.145128] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 623.145263] env[62499]: INFO nova.compute.manager [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Took 0.04 seconds to destroy the instance on the hypervisor. [ 623.145432] env[62499]: DEBUG oslo.service.loopingcall [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 623.145646] env[62499]: DEBUG nova.compute.manager [-] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 623.145736] env[62499]: DEBUG nova.network.neutron [-] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 623.162533] env[62499]: DEBUG nova.network.neutron [-] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 623.218197] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.024s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 623.219734] env[62499]: ERROR nova.compute.manager [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 6eb83bc1-e5fd-4aba-bb63-fc8d979dbabf, please check neutron logs for more information. [ 623.219734] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Traceback (most recent call last): [ 623.219734] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 623.219734] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] self.driver.spawn(context, instance, image_meta, [ 623.219734] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 623.219734] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] self._vmops.spawn(context, instance, image_meta, injected_files, [ 623.219734] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 623.219734] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] vm_ref = self.build_virtual_machine(instance, [ 623.219734] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 623.219734] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] vif_infos = vmwarevif.get_vif_info(self._session, [ 623.219734] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 623.220107] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] for vif in network_info: [ 623.220107] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 623.220107] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] return self._sync_wrapper(fn, *args, **kwargs) [ 623.220107] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 623.220107] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] self.wait() [ 623.220107] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 623.220107] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] self[:] = self._gt.wait() [ 623.220107] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 623.220107] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] return self._exit_event.wait() [ 623.220107] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 623.220107] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] result = hub.switch() [ 623.220107] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 623.220107] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] return self.greenlet.switch() [ 623.220458] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 623.220458] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] result = function(*args, **kwargs) [ 623.220458] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 623.220458] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] return func(*args, **kwargs) [ 623.220458] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 623.220458] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] raise e [ 623.220458] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 623.220458] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] nwinfo = self.network_api.allocate_for_instance( [ 623.220458] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 623.220458] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] created_port_ids = self._update_ports_for_instance( [ 623.220458] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 623.220458] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] with excutils.save_and_reraise_exception(): [ 623.220458] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 623.220853] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] self.force_reraise() [ 623.220853] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 623.220853] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] raise self.value [ 623.220853] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 623.220853] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] updated_port = self._update_port( [ 623.220853] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 623.220853] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] _ensure_no_port_binding_failure(port) [ 623.220853] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 623.220853] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] raise exception.PortBindingFailed(port_id=port['id']) [ 623.220853] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] nova.exception.PortBindingFailed: Binding failed for port 6eb83bc1-e5fd-4aba-bb63-fc8d979dbabf, please check neutron logs for more information. [ 623.220853] env[62499]: ERROR nova.compute.manager [instance: 7661165d-2731-456e-af25-da13c68fd9fb] [ 623.221167] env[62499]: DEBUG nova.compute.utils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Binding failed for port 6eb83bc1-e5fd-4aba-bb63-fc8d979dbabf, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 623.221693] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 20.103s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 623.223489] env[62499]: INFO nova.compute.claims [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 623.229019] env[62499]: DEBUG nova.compute.manager [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Build of instance 7661165d-2731-456e-af25-da13c68fd9fb was re-scheduled: Binding failed for port 6eb83bc1-e5fd-4aba-bb63-fc8d979dbabf, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 623.229019] env[62499]: DEBUG nova.compute.manager [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 623.229019] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Acquiring lock "refresh_cache-7661165d-2731-456e-af25-da13c68fd9fb" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 623.229019] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Acquired lock "refresh_cache-7661165d-2731-456e-af25-da13c68fd9fb" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 623.229545] env[62499]: DEBUG nova.network.neutron [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 623.498750] env[62499]: INFO nova.compute.manager [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733] Took 1.03 seconds to deallocate network for instance. [ 624.190846] env[62499]: DEBUG nova.network.neutron [-] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 624.200996] env[62499]: DEBUG nova.compute.manager [req-0695bab0-45c0-4321-b64e-650930d33ec2 req-55563b70-8015-49f2-9345-89b266057a2a service nova] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Received event network-vif-deleted-c853a44f-5ddf-4d9b-b501-5cf4d5772cd4 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 624.213897] env[62499]: DEBUG nova.network.neutron [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 624.231117] env[62499]: DEBUG nova.network.neutron [req-29530884-fa26-4501-a0ea-5d06e1896d9b req-6632173c-6ed4-4ba8-9940-0304d7a3defd service nova] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 624.286868] env[62499]: DEBUG nova.network.neutron [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 624.345799] env[62499]: DEBUG nova.network.neutron [req-29530884-fa26-4501-a0ea-5d06e1896d9b req-6632173c-6ed4-4ba8-9940-0304d7a3defd service nova] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 624.704846] env[62499]: INFO nova.compute.manager [-] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Took 1.56 seconds to deallocate network for instance. [ 624.708099] env[62499]: DEBUG nova.compute.claims [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 624.708405] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 624.774580] env[62499]: INFO nova.scheduler.client.report [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Deleted allocations for instance 0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733 [ 624.789959] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Releasing lock "refresh_cache-7661165d-2731-456e-af25-da13c68fd9fb" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 624.790462] env[62499]: DEBUG nova.compute.manager [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 624.790636] env[62499]: DEBUG nova.compute.manager [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 624.790808] env[62499]: DEBUG nova.network.neutron [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 624.848173] env[62499]: DEBUG oslo_concurrency.lockutils [req-29530884-fa26-4501-a0ea-5d06e1896d9b req-6632173c-6ed4-4ba8-9940-0304d7a3defd service nova] Releasing lock "refresh_cache-81b1d279-dfad-4a2c-bcfb-043b13d77af6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 624.941888] env[62499]: DEBUG nova.network.neutron [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 625.091020] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2b3f64a8-0a3b-4407-bef5-3626040d9ccc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.098996] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-097722a4-a029-48d3-bd02-939e1e501f8b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.130293] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e304d12c-8332-49a0-9e13-43295a731d0a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.138662] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-297041e8-a433-4e8a-aa07-c7764aa83b53 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.154405] env[62499]: DEBUG nova.compute.provider_tree [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 625.293751] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0468becc-c67e-4f75-9c76-8f47c0479491 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Lock "0361a54c-f2ac-4f07-8cf5-4ecdc2c1f733" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 67.209s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 625.446011] env[62499]: DEBUG nova.network.neutron [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 625.657883] env[62499]: DEBUG nova.scheduler.client.report [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 625.796356] env[62499]: DEBUG nova.compute.manager [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 625.950292] env[62499]: INFO nova.compute.manager [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] [instance: 7661165d-2731-456e-af25-da13c68fd9fb] Took 1.16 seconds to deallocate network for instance. [ 626.166204] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.942s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 626.166204] env[62499]: DEBUG nova.compute.manager [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 626.167511] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 20.479s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 626.321214] env[62499]: DEBUG oslo_concurrency.lockutils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 626.675202] env[62499]: DEBUG nova.compute.utils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 626.682355] env[62499]: DEBUG nova.compute.manager [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 626.685450] env[62499]: DEBUG nova.network.neutron [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 626.766069] env[62499]: DEBUG nova.policy [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f05975529be14a03b5f165f134a770d4', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'a5a2a6c3b01640eba0a5b5612711385d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 626.995122] env[62499]: INFO nova.scheduler.client.report [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Deleted allocations for instance 7661165d-2731-456e-af25-da13c68fd9fb [ 627.141841] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-243d7731-51bf-45ad-ac77-1c6ddf33427f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 627.148495] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2bad8a5e-99cc-4d67-adf7-f4f096084910 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 627.181263] env[62499]: DEBUG nova.network.neutron [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Successfully created port: 29b75515-596f-4fac-b874-1afada623805 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 627.183823] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b1a2456-6a07-40e2-bd34-8d91aa83b31d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 627.186532] env[62499]: DEBUG nova.compute.manager [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 627.195460] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7a9981d-f7b1-4ac8-96be-da8fcb62285a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 627.209628] env[62499]: DEBUG nova.compute.provider_tree [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 627.509365] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d0d58171-f33c-4213-a5d5-cb3cfb0fecd9 tempest-ServerDiagnosticsNegativeTest-1650443261 tempest-ServerDiagnosticsNegativeTest-1650443261-project-member] Lock "7661165d-2731-456e-af25-da13c68fd9fb" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 67.229s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 627.713359] env[62499]: DEBUG nova.scheduler.client.report [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 628.014022] env[62499]: DEBUG nova.compute.manager [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 628.198541] env[62499]: DEBUG nova.compute.manager [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 628.225018] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.057s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 628.225018] env[62499]: ERROR nova.compute.manager [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 965c7a6a-7b9b-49b0-8f72-a6eea9ca2675, please check neutron logs for more information. [ 628.225018] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Traceback (most recent call last): [ 628.225018] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 628.225018] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] self.driver.spawn(context, instance, image_meta, [ 628.225018] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 628.225018] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 628.225018] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 628.225018] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] vm_ref = self.build_virtual_machine(instance, [ 628.225296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 628.225296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] vif_infos = vmwarevif.get_vif_info(self._session, [ 628.225296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 628.225296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] for vif in network_info: [ 628.225296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 628.225296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] return self._sync_wrapper(fn, *args, **kwargs) [ 628.225296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 628.225296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] self.wait() [ 628.225296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 628.225296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] self[:] = self._gt.wait() [ 628.225296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 628.225296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] return self._exit_event.wait() [ 628.225296] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 628.225578] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] result = hub.switch() [ 628.225578] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 628.225578] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] return self.greenlet.switch() [ 628.225578] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 628.225578] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] result = function(*args, **kwargs) [ 628.225578] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 628.225578] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] return func(*args, **kwargs) [ 628.225578] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 628.225578] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] raise e [ 628.225578] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 628.225578] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] nwinfo = self.network_api.allocate_for_instance( [ 628.225578] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 628.225578] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] created_port_ids = self._update_ports_for_instance( [ 628.225906] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 628.225906] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] with excutils.save_and_reraise_exception(): [ 628.225906] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 628.225906] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] self.force_reraise() [ 628.225906] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 628.225906] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] raise self.value [ 628.225906] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 628.225906] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] updated_port = self._update_port( [ 628.225906] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 628.225906] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] _ensure_no_port_binding_failure(port) [ 628.225906] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 628.225906] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] raise exception.PortBindingFailed(port_id=port['id']) [ 628.226247] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] nova.exception.PortBindingFailed: Binding failed for port 965c7a6a-7b9b-49b0-8f72-a6eea9ca2675, please check neutron logs for more information. [ 628.226247] env[62499]: ERROR nova.compute.manager [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] [ 628.227376] env[62499]: DEBUG nova.compute.utils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Binding failed for port 965c7a6a-7b9b-49b0-8f72-a6eea9ca2675, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 628.229146] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 19.493s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 628.230798] env[62499]: INFO nova.compute.claims [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 628.236800] env[62499]: DEBUG nova.virt.hardware [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 628.237034] env[62499]: DEBUG nova.virt.hardware [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 628.237197] env[62499]: DEBUG nova.virt.hardware [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 628.237379] env[62499]: DEBUG nova.virt.hardware [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 628.237525] env[62499]: DEBUG nova.virt.hardware [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 628.237694] env[62499]: DEBUG nova.virt.hardware [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 628.237923] env[62499]: DEBUG nova.virt.hardware [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 628.242237] env[62499]: DEBUG nova.virt.hardware [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 628.242505] env[62499]: DEBUG nova.virt.hardware [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 628.242716] env[62499]: DEBUG nova.virt.hardware [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 628.242936] env[62499]: DEBUG nova.virt.hardware [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 628.243547] env[62499]: DEBUG nova.compute.manager [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Build of instance 037217df-a3ed-4cef-9478-a9cc576a2f7d was re-scheduled: Binding failed for port 965c7a6a-7b9b-49b0-8f72-a6eea9ca2675, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 628.244077] env[62499]: DEBUG nova.compute.manager [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 628.244585] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Acquiring lock "refresh_cache-037217df-a3ed-4cef-9478-a9cc576a2f7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 628.245457] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Acquired lock "refresh_cache-037217df-a3ed-4cef-9478-a9cc576a2f7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 628.245875] env[62499]: DEBUG nova.network.neutron [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 628.247518] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc5177fd-fc49-4210-8321-478378387440 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 628.258953] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8a23c90d-6c82-424a-9967-437453271746 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 628.539142] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 628.596775] env[62499]: ERROR nova.compute.manager [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 29b75515-596f-4fac-b874-1afada623805, please check neutron logs for more information. [ 628.596775] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 628.596775] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 628.596775] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 628.596775] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 628.596775] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 628.596775] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 628.596775] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 628.596775] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 628.596775] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 628.596775] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 628.596775] env[62499]: ERROR nova.compute.manager raise self.value [ 628.596775] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 628.596775] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 628.596775] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 628.596775] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 628.597584] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 628.597584] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 628.597584] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 29b75515-596f-4fac-b874-1afada623805, please check neutron logs for more information. [ 628.597584] env[62499]: ERROR nova.compute.manager [ 628.597584] env[62499]: Traceback (most recent call last): [ 628.597584] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 628.597584] env[62499]: listener.cb(fileno) [ 628.597584] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 628.597584] env[62499]: result = function(*args, **kwargs) [ 628.597584] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 628.597584] env[62499]: return func(*args, **kwargs) [ 628.597584] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 628.597584] env[62499]: raise e [ 628.597584] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 628.597584] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 628.597584] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 628.597584] env[62499]: created_port_ids = self._update_ports_for_instance( [ 628.597584] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 628.597584] env[62499]: with excutils.save_and_reraise_exception(): [ 628.597584] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 628.597584] env[62499]: self.force_reraise() [ 628.597584] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 628.597584] env[62499]: raise self.value [ 628.597584] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 628.597584] env[62499]: updated_port = self._update_port( [ 628.597584] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 628.597584] env[62499]: _ensure_no_port_binding_failure(port) [ 628.597584] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 628.597584] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 628.598717] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 29b75515-596f-4fac-b874-1afada623805, please check neutron logs for more information. [ 628.598717] env[62499]: Removing descriptor: 15 [ 628.598717] env[62499]: ERROR nova.compute.manager [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 29b75515-596f-4fac-b874-1afada623805, please check neutron logs for more information. [ 628.598717] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Traceback (most recent call last): [ 628.598717] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 628.598717] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] yield resources [ 628.598717] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 628.598717] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] self.driver.spawn(context, instance, image_meta, [ 628.598717] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 628.598717] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 628.598717] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 628.598717] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] vm_ref = self.build_virtual_machine(instance, [ 628.599168] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 628.599168] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] vif_infos = vmwarevif.get_vif_info(self._session, [ 628.599168] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 628.599168] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] for vif in network_info: [ 628.599168] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 628.599168] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] return self._sync_wrapper(fn, *args, **kwargs) [ 628.599168] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 628.599168] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] self.wait() [ 628.599168] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 628.599168] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] self[:] = self._gt.wait() [ 628.599168] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 628.599168] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] return self._exit_event.wait() [ 628.599168] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 628.599616] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] result = hub.switch() [ 628.599616] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 628.599616] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] return self.greenlet.switch() [ 628.599616] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 628.599616] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] result = function(*args, **kwargs) [ 628.599616] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 628.599616] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] return func(*args, **kwargs) [ 628.599616] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 628.599616] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] raise e [ 628.599616] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 628.599616] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] nwinfo = self.network_api.allocate_for_instance( [ 628.599616] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 628.599616] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] created_port_ids = self._update_ports_for_instance( [ 628.599896] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 628.599896] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] with excutils.save_and_reraise_exception(): [ 628.599896] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 628.599896] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] self.force_reraise() [ 628.599896] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 628.599896] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] raise self.value [ 628.599896] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 628.599896] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] updated_port = self._update_port( [ 628.599896] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 628.599896] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] _ensure_no_port_binding_failure(port) [ 628.599896] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 628.599896] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] raise exception.PortBindingFailed(port_id=port['id']) [ 628.600162] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] nova.exception.PortBindingFailed: Binding failed for port 29b75515-596f-4fac-b874-1afada623805, please check neutron logs for more information. [ 628.600162] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] [ 628.600162] env[62499]: INFO nova.compute.manager [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Terminating instance [ 628.600376] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Acquiring lock "refresh_cache-ccce765d-8df9-4837-b166-71e41bea6de7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 628.600497] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Acquired lock "refresh_cache-ccce765d-8df9-4837-b166-71e41bea6de7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 628.600660] env[62499]: DEBUG nova.network.neutron [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 628.679236] env[62499]: DEBUG nova.compute.manager [req-d3d2bad3-97c5-4513-b2c3-d3dd3d25f4b9 req-a6ff4adb-5fe3-478f-8a72-1f5a25519b04 service nova] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Received event network-changed-29b75515-596f-4fac-b874-1afada623805 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 628.679437] env[62499]: DEBUG nova.compute.manager [req-d3d2bad3-97c5-4513-b2c3-d3dd3d25f4b9 req-a6ff4adb-5fe3-478f-8a72-1f5a25519b04 service nova] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Refreshing instance network info cache due to event network-changed-29b75515-596f-4fac-b874-1afada623805. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 628.679693] env[62499]: DEBUG oslo_concurrency.lockutils [req-d3d2bad3-97c5-4513-b2c3-d3dd3d25f4b9 req-a6ff4adb-5fe3-478f-8a72-1f5a25519b04 service nova] Acquiring lock "refresh_cache-ccce765d-8df9-4837-b166-71e41bea6de7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 628.723857] env[62499]: DEBUG oslo_concurrency.lockutils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Acquiring lock "78a83324-36da-457e-a78d-c82d0a722015" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 628.723857] env[62499]: DEBUG oslo_concurrency.lockutils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Lock "78a83324-36da-457e-a78d-c82d0a722015" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 628.787892] env[62499]: DEBUG nova.network.neutron [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 628.877627] env[62499]: DEBUG nova.network.neutron [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 629.123850] env[62499]: DEBUG nova.network.neutron [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 629.190019] env[62499]: DEBUG nova.network.neutron [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 629.381933] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Releasing lock "refresh_cache-037217df-a3ed-4cef-9478-a9cc576a2f7d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 629.382079] env[62499]: DEBUG nova.compute.manager [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 629.382256] env[62499]: DEBUG nova.compute.manager [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 629.382454] env[62499]: DEBUG nova.network.neutron [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 629.412089] env[62499]: DEBUG nova.network.neutron [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 629.521269] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Acquiring lock "dee17e73-22db-48ff-b0b0-ec7bec850c48" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 629.521349] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Lock "dee17e73-22db-48ff-b0b0-ec7bec850c48" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 629.603382] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8cfc532-bde5-48ba-90e1-de28c54bfde6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 629.611038] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72dae86d-d63f-4911-9c67-344226bf2eab {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 629.642194] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d9dd0989-c037-447d-8885-aeff9beb2848 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 629.649672] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2a48538-149d-41bd-b33e-47f0a3dac1a7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 629.663048] env[62499]: DEBUG nova.compute.provider_tree [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 629.693423] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Releasing lock "refresh_cache-ccce765d-8df9-4837-b166-71e41bea6de7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 629.693816] env[62499]: DEBUG nova.compute.manager [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 629.694017] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 629.694338] env[62499]: DEBUG oslo_concurrency.lockutils [req-d3d2bad3-97c5-4513-b2c3-d3dd3d25f4b9 req-a6ff4adb-5fe3-478f-8a72-1f5a25519b04 service nova] Acquired lock "refresh_cache-ccce765d-8df9-4837-b166-71e41bea6de7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 629.694530] env[62499]: DEBUG nova.network.neutron [req-d3d2bad3-97c5-4513-b2c3-d3dd3d25f4b9 req-a6ff4adb-5fe3-478f-8a72-1f5a25519b04 service nova] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Refreshing network info cache for port 29b75515-596f-4fac-b874-1afada623805 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 629.695574] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-aaad7946-1fe0-4951-ad26-1402314c7ede {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 629.705092] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-414c36f2-c10c-4f05-9e4e-b77f43da466e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 629.730555] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ccce765d-8df9-4837-b166-71e41bea6de7 could not be found. [ 629.730790] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 629.730867] env[62499]: INFO nova.compute.manager [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Took 0.04 seconds to destroy the instance on the hypervisor. [ 629.731137] env[62499]: DEBUG oslo.service.loopingcall [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 629.731342] env[62499]: DEBUG nova.compute.manager [-] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 629.731435] env[62499]: DEBUG nova.network.neutron [-] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 629.753916] env[62499]: DEBUG nova.network.neutron [-] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 629.916356] env[62499]: DEBUG nova.network.neutron [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 630.166906] env[62499]: DEBUG nova.scheduler.client.report [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 630.214946] env[62499]: DEBUG nova.network.neutron [req-d3d2bad3-97c5-4513-b2c3-d3dd3d25f4b9 req-a6ff4adb-5fe3-478f-8a72-1f5a25519b04 service nova] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 630.260480] env[62499]: DEBUG nova.network.neutron [-] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 630.307116] env[62499]: DEBUG nova.network.neutron [req-d3d2bad3-97c5-4513-b2c3-d3dd3d25f4b9 req-a6ff4adb-5fe3-478f-8a72-1f5a25519b04 service nova] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 630.419640] env[62499]: INFO nova.compute.manager [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] [instance: 037217df-a3ed-4cef-9478-a9cc576a2f7d] Took 1.04 seconds to deallocate network for instance. [ 630.672790] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.444s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 630.673711] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 630.677082] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 18.852s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 630.706436] env[62499]: DEBUG nova.compute.manager [req-e57b5703-7b0c-4afc-9f20-c0e374d43f6c req-142987e5-e34e-4816-9ab3-b0dd6bb3a33f service nova] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Received event network-vif-deleted-29b75515-596f-4fac-b874-1afada623805 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 630.763676] env[62499]: INFO nova.compute.manager [-] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Took 1.03 seconds to deallocate network for instance. [ 630.768304] env[62499]: DEBUG nova.compute.claims [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 630.768702] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 630.810077] env[62499]: DEBUG oslo_concurrency.lockutils [req-d3d2bad3-97c5-4513-b2c3-d3dd3d25f4b9 req-a6ff4adb-5fe3-478f-8a72-1f5a25519b04 service nova] Releasing lock "refresh_cache-ccce765d-8df9-4837-b166-71e41bea6de7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 631.182407] env[62499]: DEBUG nova.compute.utils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 631.188018] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 631.188080] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 631.248197] env[62499]: DEBUG nova.policy [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd896dc782fdf4c13b943c0336d255452', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7fa50ab2c6ba4659afee1f146822a981', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 631.459186] env[62499]: INFO nova.scheduler.client.report [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Deleted allocations for instance 037217df-a3ed-4cef-9478-a9cc576a2f7d [ 631.575565] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d509d02-bff5-4733-9f39-d6d6c21ba454 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 631.587240] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85c2370f-138d-484c-b3b0-4dfd9dbdbb3e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 631.619635] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ac31b6f-a3ab-484a-961e-c83c98e90b09 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 631.627870] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-314ac597-1244-4469-957e-c44f54695ae4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 631.642624] env[62499]: DEBUG nova.compute.provider_tree [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 631.690944] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 631.852141] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Successfully created port: fc688df4-d52b-4751-a8c0-854113d5e094 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 631.969495] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4657df64-e839-4424-bf76-08371706d845 tempest-ServerTagsTestJSON-2002162229 tempest-ServerTagsTestJSON-2002162229-project-member] Lock "037217df-a3ed-4cef-9478-a9cc576a2f7d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 69.349s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 632.148289] env[62499]: DEBUG nova.scheduler.client.report [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 632.473176] env[62499]: DEBUG nova.compute.manager [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 632.658352] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.978s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 632.658352] env[62499]: ERROR nova.compute.manager [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d22afec3-7f0e-413f-b7ee-8529d26f4c99, please check neutron logs for more information. [ 632.658352] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Traceback (most recent call last): [ 632.658352] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 632.658352] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] self.driver.spawn(context, instance, image_meta, [ 632.658352] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 632.658352] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] self._vmops.spawn(context, instance, image_meta, injected_files, [ 632.658352] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 632.658352] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] vm_ref = self.build_virtual_machine(instance, [ 632.658615] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 632.658615] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] vif_infos = vmwarevif.get_vif_info(self._session, [ 632.658615] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 632.658615] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] for vif in network_info: [ 632.658615] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 632.658615] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] return self._sync_wrapper(fn, *args, **kwargs) [ 632.658615] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 632.658615] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] self.wait() [ 632.658615] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 632.658615] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] self[:] = self._gt.wait() [ 632.658615] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 632.658615] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] return self._exit_event.wait() [ 632.658615] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 632.659031] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] current.throw(*self._exc) [ 632.659031] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 632.659031] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] result = function(*args, **kwargs) [ 632.659031] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 632.659031] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] return func(*args, **kwargs) [ 632.659031] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 632.659031] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] raise e [ 632.659031] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 632.659031] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] nwinfo = self.network_api.allocate_for_instance( [ 632.659031] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 632.659031] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] created_port_ids = self._update_ports_for_instance( [ 632.659031] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 632.659031] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] with excutils.save_and_reraise_exception(): [ 632.659357] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 632.659357] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] self.force_reraise() [ 632.659357] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 632.659357] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] raise self.value [ 632.659357] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 632.659357] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] updated_port = self._update_port( [ 632.659357] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 632.659357] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] _ensure_no_port_binding_failure(port) [ 632.659357] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 632.659357] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] raise exception.PortBindingFailed(port_id=port['id']) [ 632.659357] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] nova.exception.PortBindingFailed: Binding failed for port d22afec3-7f0e-413f-b7ee-8529d26f4c99, please check neutron logs for more information. [ 632.659357] env[62499]: ERROR nova.compute.manager [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] [ 632.659663] env[62499]: DEBUG nova.compute.utils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Binding failed for port d22afec3-7f0e-413f-b7ee-8529d26f4c99, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 632.659663] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.132s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 632.663517] env[62499]: INFO nova.compute.claims [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 632.667332] env[62499]: DEBUG nova.compute.manager [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Build of instance 9160ca3a-2aa2-4cfc-9900-3e4c770d2657 was re-scheduled: Binding failed for port d22afec3-7f0e-413f-b7ee-8529d26f4c99, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 632.668154] env[62499]: DEBUG nova.compute.manager [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 632.668389] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Acquiring lock "refresh_cache-9160ca3a-2aa2-4cfc-9900-3e4c770d2657" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 632.668557] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Acquired lock "refresh_cache-9160ca3a-2aa2-4cfc-9900-3e4c770d2657" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 632.668745] env[62499]: DEBUG nova.network.neutron [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 632.697609] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 632.736021] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 632.736021] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 632.736021] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 632.736230] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 632.736558] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 632.736892] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 632.737385] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 632.737760] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 632.740024] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 632.740024] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 632.740024] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 632.740024] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6fc7ca6-50ee-4312-95e9-324860fd802f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 632.749448] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-302faf54-3715-41e8-ad42-ca153635a6c1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 633.002853] env[62499]: DEBUG oslo_concurrency.lockutils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 633.205175] env[62499]: DEBUG nova.network.neutron [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 633.429536] env[62499]: DEBUG nova.network.neutron [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 633.547707] env[62499]: DEBUG nova.compute.manager [req-8f5f4088-ec7f-47d9-90ff-aadd5101b706 req-4c869ca8-07eb-43d4-affc-083477fe2da6 service nova] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Received event network-changed-fc688df4-d52b-4751-a8c0-854113d5e094 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 633.547797] env[62499]: DEBUG nova.compute.manager [req-8f5f4088-ec7f-47d9-90ff-aadd5101b706 req-4c869ca8-07eb-43d4-affc-083477fe2da6 service nova] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Refreshing instance network info cache due to event network-changed-fc688df4-d52b-4751-a8c0-854113d5e094. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 633.548451] env[62499]: DEBUG oslo_concurrency.lockutils [req-8f5f4088-ec7f-47d9-90ff-aadd5101b706 req-4c869ca8-07eb-43d4-affc-083477fe2da6 service nova] Acquiring lock "refresh_cache-287c3404-9fc1-4369-b9a4-17da918bf78d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 633.548451] env[62499]: DEBUG oslo_concurrency.lockutils [req-8f5f4088-ec7f-47d9-90ff-aadd5101b706 req-4c869ca8-07eb-43d4-affc-083477fe2da6 service nova] Acquired lock "refresh_cache-287c3404-9fc1-4369-b9a4-17da918bf78d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 633.548451] env[62499]: DEBUG nova.network.neutron [req-8f5f4088-ec7f-47d9-90ff-aadd5101b706 req-4c869ca8-07eb-43d4-affc-083477fe2da6 service nova] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Refreshing network info cache for port fc688df4-d52b-4751-a8c0-854113d5e094 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 633.784932] env[62499]: ERROR nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port fc688df4-d52b-4751-a8c0-854113d5e094, please check neutron logs for more information. [ 633.784932] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 633.784932] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 633.784932] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 633.784932] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 633.784932] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 633.784932] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 633.784932] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 633.784932] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 633.784932] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 633.784932] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 633.784932] env[62499]: ERROR nova.compute.manager raise self.value [ 633.784932] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 633.784932] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 633.784932] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 633.784932] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 633.785768] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 633.785768] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 633.785768] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port fc688df4-d52b-4751-a8c0-854113d5e094, please check neutron logs for more information. [ 633.785768] env[62499]: ERROR nova.compute.manager [ 633.785768] env[62499]: Traceback (most recent call last): [ 633.785768] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 633.785768] env[62499]: listener.cb(fileno) [ 633.785768] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 633.785768] env[62499]: result = function(*args, **kwargs) [ 633.785768] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 633.785768] env[62499]: return func(*args, **kwargs) [ 633.785768] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 633.785768] env[62499]: raise e [ 633.785768] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 633.785768] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 633.785768] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 633.785768] env[62499]: created_port_ids = self._update_ports_for_instance( [ 633.785768] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 633.785768] env[62499]: with excutils.save_and_reraise_exception(): [ 633.785768] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 633.785768] env[62499]: self.force_reraise() [ 633.785768] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 633.785768] env[62499]: raise self.value [ 633.785768] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 633.785768] env[62499]: updated_port = self._update_port( [ 633.785768] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 633.785768] env[62499]: _ensure_no_port_binding_failure(port) [ 633.785768] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 633.785768] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 633.788600] env[62499]: nova.exception.PortBindingFailed: Binding failed for port fc688df4-d52b-4751-a8c0-854113d5e094, please check neutron logs for more information. [ 633.788600] env[62499]: Removing descriptor: 15 [ 633.788600] env[62499]: ERROR nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port fc688df4-d52b-4751-a8c0-854113d5e094, please check neutron logs for more information. [ 633.788600] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Traceback (most recent call last): [ 633.788600] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 633.788600] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] yield resources [ 633.788600] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 633.788600] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] self.driver.spawn(context, instance, image_meta, [ 633.788600] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 633.788600] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 633.788600] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 633.788600] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] vm_ref = self.build_virtual_machine(instance, [ 633.789057] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 633.789057] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] vif_infos = vmwarevif.get_vif_info(self._session, [ 633.789057] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 633.789057] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] for vif in network_info: [ 633.789057] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 633.789057] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] return self._sync_wrapper(fn, *args, **kwargs) [ 633.789057] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 633.789057] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] self.wait() [ 633.789057] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 633.789057] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] self[:] = self._gt.wait() [ 633.789057] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 633.789057] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] return self._exit_event.wait() [ 633.789057] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 633.789362] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] result = hub.switch() [ 633.789362] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 633.789362] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] return self.greenlet.switch() [ 633.789362] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 633.789362] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] result = function(*args, **kwargs) [ 633.789362] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 633.789362] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] return func(*args, **kwargs) [ 633.789362] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 633.789362] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] raise e [ 633.789362] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 633.789362] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] nwinfo = self.network_api.allocate_for_instance( [ 633.789362] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 633.789362] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] created_port_ids = self._update_ports_for_instance( [ 633.789776] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 633.789776] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] with excutils.save_and_reraise_exception(): [ 633.789776] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 633.789776] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] self.force_reraise() [ 633.789776] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 633.789776] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] raise self.value [ 633.789776] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 633.789776] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] updated_port = self._update_port( [ 633.789776] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 633.789776] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] _ensure_no_port_binding_failure(port) [ 633.789776] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 633.789776] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] raise exception.PortBindingFailed(port_id=port['id']) [ 633.790774] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] nova.exception.PortBindingFailed: Binding failed for port fc688df4-d52b-4751-a8c0-854113d5e094, please check neutron logs for more information. [ 633.790774] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] [ 633.790774] env[62499]: INFO nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Terminating instance [ 633.790774] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "refresh_cache-287c3404-9fc1-4369-b9a4-17da918bf78d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 633.932016] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Releasing lock "refresh_cache-9160ca3a-2aa2-4cfc-9900-3e4c770d2657" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 633.932208] env[62499]: DEBUG nova.compute.manager [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 633.932395] env[62499]: DEBUG nova.compute.manager [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 633.932562] env[62499]: DEBUG nova.network.neutron [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 633.963065] env[62499]: DEBUG nova.network.neutron [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 634.074354] env[62499]: DEBUG nova.network.neutron [req-8f5f4088-ec7f-47d9-90ff-aadd5101b706 req-4c869ca8-07eb-43d4-affc-083477fe2da6 service nova] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 634.106103] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc5fdc69-eef0-4b9b-8478-5adac22db426 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.116268] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94e83a26-8529-4fff-8348-928c5abc4ea7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.151092] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b2095e8-4a7f-4dd9-aa67-86cfb6c4020b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.158955] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c3fcff0-cb42-45e9-ae19-6622809fc1c7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.177047] env[62499]: DEBUG nova.compute.provider_tree [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 634.194864] env[62499]: DEBUG nova.network.neutron [req-8f5f4088-ec7f-47d9-90ff-aadd5101b706 req-4c869ca8-07eb-43d4-affc-083477fe2da6 service nova] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 634.471914] env[62499]: DEBUG nova.network.neutron [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 634.482501] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 634.482918] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 634.680221] env[62499]: DEBUG nova.scheduler.client.report [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 634.697797] env[62499]: DEBUG oslo_concurrency.lockutils [req-8f5f4088-ec7f-47d9-90ff-aadd5101b706 req-4c869ca8-07eb-43d4-affc-083477fe2da6 service nova] Releasing lock "refresh_cache-287c3404-9fc1-4369-b9a4-17da918bf78d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 634.697797] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquired lock "refresh_cache-287c3404-9fc1-4369-b9a4-17da918bf78d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 634.697797] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 634.975332] env[62499]: INFO nova.compute.manager [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] [instance: 9160ca3a-2aa2-4cfc-9900-3e4c770d2657] Took 1.04 seconds to deallocate network for instance. [ 634.991162] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 634.991162] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Starting heal instance info cache {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9928}} [ 634.991162] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Rebuilding the list of instances to heal {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 635.186664] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.529s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 635.187218] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 635.190770] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.817s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 635.192155] env[62499]: INFO nova.compute.claims [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 635.226730] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 635.313018] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 635.495289] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 635.495289] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 635.495289] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 635.495289] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 635.495289] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Didn't find any instances for network info cache update. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10014}} [ 635.495289] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 635.495638] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 635.495638] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 635.495638] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 635.495638] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 635.495638] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 635.495638] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62499) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10547}} [ 635.495835] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager.update_available_resource {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 635.595355] env[62499]: DEBUG nova.compute.manager [req-c0402f1f-eb05-490e-80f2-d1f7850d5095 req-23631007-bd2f-4f44-9ecb-df78e030ea36 service nova] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Received event network-vif-deleted-fc688df4-d52b-4751-a8c0-854113d5e094 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 635.701447] env[62499]: DEBUG nova.compute.utils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 635.703122] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 635.703409] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 635.793096] env[62499]: DEBUG nova.policy [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd896dc782fdf4c13b943c0336d255452', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7fa50ab2c6ba4659afee1f146822a981', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 635.818987] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Releasing lock "refresh_cache-287c3404-9fc1-4369-b9a4-17da918bf78d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 635.818987] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 635.818987] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 635.818987] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-74f69e07-a525-456e-a7b9-16a589abe908 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.832551] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7f2116c-7e4c-4721-b0c0-bdd4c17802b1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.857268] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 287c3404-9fc1-4369-b9a4-17da918bf78d could not be found. [ 635.857499] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 635.858241] env[62499]: INFO nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Took 0.04 seconds to destroy the instance on the hypervisor. [ 635.858241] env[62499]: DEBUG oslo.service.loopingcall [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 635.859166] env[62499]: DEBUG nova.compute.manager [-] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 635.859166] env[62499]: DEBUG nova.network.neutron [-] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 635.899460] env[62499]: DEBUG nova.network.neutron [-] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 635.999185] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 636.014147] env[62499]: INFO nova.scheduler.client.report [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Deleted allocations for instance 9160ca3a-2aa2-4cfc-9900-3e4c770d2657 [ 636.163067] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Successfully created port: 77d6d7dd-312e-4215-8183-65f7e69ab012 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 636.212436] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 636.404910] env[62499]: DEBUG nova.network.neutron [-] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 636.522849] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3cc5e0bb-b5cb-47b7-a1bb-5623b10491b4 tempest-ImagesNegativeTestJSON-1203360277 tempest-ImagesNegativeTestJSON-1203360277-project-member] Lock "9160ca3a-2aa2-4cfc-9900-3e4c770d2657" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 69.648s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 636.640185] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9881ed5-21a1-4012-8c96-052da2542c26 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 636.648048] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f84070be-537e-47fa-85b9-c4be60056548 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 636.687184] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c22cf135-9549-42b5-9458-094da44c727e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 636.696434] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7bed5f50-437b-4d50-b8fd-74bb435c4ec2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 636.715755] env[62499]: DEBUG nova.compute.provider_tree [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 636.908786] env[62499]: INFO nova.compute.manager [-] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Took 1.05 seconds to deallocate network for instance. [ 636.911769] env[62499]: DEBUG nova.compute.claims [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 636.911957] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 637.027358] env[62499]: DEBUG nova.compute.manager [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 637.224380] env[62499]: DEBUG nova.scheduler.client.report [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 637.231264] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 637.269931] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 637.270240] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 637.270397] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 637.270604] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 637.270788] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 637.270944] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 637.271433] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 637.271779] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 637.271974] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 637.272194] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 637.272380] env[62499]: DEBUG nova.virt.hardware [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 637.274110] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c462692-bd75-4a57-b278-d1b03fd9b54e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 637.287424] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-31ee1a97-a488-4940-ac3d-fa7517a0f2a9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 637.562408] env[62499]: DEBUG oslo_concurrency.lockutils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 637.736519] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.546s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 637.737169] env[62499]: DEBUG nova.compute.manager [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 637.741115] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.325s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 637.745017] env[62499]: INFO nova.compute.claims [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 637.999886] env[62499]: DEBUG nova.compute.manager [req-8a377021-722b-4e8e-81c5-f64da9f6059d req-c123d3e0-faf6-45f6-a110-ffd9e9ec8e6c service nova] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Received event network-changed-77d6d7dd-312e-4215-8183-65f7e69ab012 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 637.999886] env[62499]: DEBUG nova.compute.manager [req-8a377021-722b-4e8e-81c5-f64da9f6059d req-c123d3e0-faf6-45f6-a110-ffd9e9ec8e6c service nova] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Refreshing instance network info cache due to event network-changed-77d6d7dd-312e-4215-8183-65f7e69ab012. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 637.999886] env[62499]: DEBUG oslo_concurrency.lockutils [req-8a377021-722b-4e8e-81c5-f64da9f6059d req-c123d3e0-faf6-45f6-a110-ffd9e9ec8e6c service nova] Acquiring lock "refresh_cache-6680c707-aa59-4e00-a2b1-38d465cc6497" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 637.999886] env[62499]: DEBUG oslo_concurrency.lockutils [req-8a377021-722b-4e8e-81c5-f64da9f6059d req-c123d3e0-faf6-45f6-a110-ffd9e9ec8e6c service nova] Acquired lock "refresh_cache-6680c707-aa59-4e00-a2b1-38d465cc6497" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 638.000920] env[62499]: DEBUG nova.network.neutron [req-8a377021-722b-4e8e-81c5-f64da9f6059d req-c123d3e0-faf6-45f6-a110-ffd9e9ec8e6c service nova] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Refreshing network info cache for port 77d6d7dd-312e-4215-8183-65f7e69ab012 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 638.249118] env[62499]: DEBUG nova.compute.utils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 638.254169] env[62499]: DEBUG nova.compute.manager [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 638.255747] env[62499]: DEBUG nova.network.neutron [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 638.338961] env[62499]: DEBUG nova.policy [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '34030955a6164c8f8f90b42bbf9c5551', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '2e3c5bed90a143e2baa34a7df2895fa5', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 638.396968] env[62499]: ERROR nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 77d6d7dd-312e-4215-8183-65f7e69ab012, please check neutron logs for more information. [ 638.396968] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 638.396968] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 638.396968] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 638.396968] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 638.396968] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 638.396968] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 638.396968] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 638.396968] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 638.396968] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 638.396968] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 638.396968] env[62499]: ERROR nova.compute.manager raise self.value [ 638.396968] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 638.396968] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 638.396968] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 638.396968] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 638.397424] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 638.397424] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 638.397424] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 77d6d7dd-312e-4215-8183-65f7e69ab012, please check neutron logs for more information. [ 638.397424] env[62499]: ERROR nova.compute.manager [ 638.397424] env[62499]: Traceback (most recent call last): [ 638.397424] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 638.397424] env[62499]: listener.cb(fileno) [ 638.397424] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 638.397424] env[62499]: result = function(*args, **kwargs) [ 638.397424] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 638.397424] env[62499]: return func(*args, **kwargs) [ 638.397424] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 638.397424] env[62499]: raise e [ 638.397424] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 638.397424] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 638.397424] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 638.397424] env[62499]: created_port_ids = self._update_ports_for_instance( [ 638.397424] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 638.397424] env[62499]: with excutils.save_and_reraise_exception(): [ 638.397424] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 638.397424] env[62499]: self.force_reraise() [ 638.397424] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 638.397424] env[62499]: raise self.value [ 638.397424] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 638.397424] env[62499]: updated_port = self._update_port( [ 638.397424] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 638.397424] env[62499]: _ensure_no_port_binding_failure(port) [ 638.397424] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 638.397424] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 638.398121] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 77d6d7dd-312e-4215-8183-65f7e69ab012, please check neutron logs for more information. [ 638.398121] env[62499]: Removing descriptor: 15 [ 638.398272] env[62499]: ERROR nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 77d6d7dd-312e-4215-8183-65f7e69ab012, please check neutron logs for more information. [ 638.398272] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Traceback (most recent call last): [ 638.398272] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 638.398272] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] yield resources [ 638.398272] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 638.398272] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] self.driver.spawn(context, instance, image_meta, [ 638.398272] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 638.398272] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] self._vmops.spawn(context, instance, image_meta, injected_files, [ 638.398272] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 638.398272] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] vm_ref = self.build_virtual_machine(instance, [ 638.398272] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 638.398601] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] vif_infos = vmwarevif.get_vif_info(self._session, [ 638.398601] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 638.398601] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] for vif in network_info: [ 638.398601] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 638.398601] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] return self._sync_wrapper(fn, *args, **kwargs) [ 638.398601] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 638.398601] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] self.wait() [ 638.398601] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 638.398601] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] self[:] = self._gt.wait() [ 638.398601] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 638.398601] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] return self._exit_event.wait() [ 638.398601] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 638.398601] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] result = hub.switch() [ 638.398985] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 638.398985] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] return self.greenlet.switch() [ 638.398985] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 638.398985] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] result = function(*args, **kwargs) [ 638.398985] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 638.398985] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] return func(*args, **kwargs) [ 638.398985] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 638.398985] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] raise e [ 638.398985] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 638.398985] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] nwinfo = self.network_api.allocate_for_instance( [ 638.398985] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 638.398985] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] created_port_ids = self._update_ports_for_instance( [ 638.398985] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 638.399375] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] with excutils.save_and_reraise_exception(): [ 638.399375] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 638.399375] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] self.force_reraise() [ 638.399375] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 638.399375] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] raise self.value [ 638.399375] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 638.399375] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] updated_port = self._update_port( [ 638.399375] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 638.399375] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] _ensure_no_port_binding_failure(port) [ 638.399375] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 638.399375] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] raise exception.PortBindingFailed(port_id=port['id']) [ 638.399375] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] nova.exception.PortBindingFailed: Binding failed for port 77d6d7dd-312e-4215-8183-65f7e69ab012, please check neutron logs for more information. [ 638.399375] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] [ 638.399768] env[62499]: INFO nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Terminating instance [ 638.400755] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "refresh_cache-6680c707-aa59-4e00-a2b1-38d465cc6497" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 638.542175] env[62499]: DEBUG nova.network.neutron [req-8a377021-722b-4e8e-81c5-f64da9f6059d req-c123d3e0-faf6-45f6-a110-ffd9e9ec8e6c service nova] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 638.756102] env[62499]: DEBUG nova.compute.manager [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 638.767403] env[62499]: DEBUG nova.network.neutron [req-8a377021-722b-4e8e-81c5-f64da9f6059d req-c123d3e0-faf6-45f6-a110-ffd9e9ec8e6c service nova] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 639.110995] env[62499]: DEBUG nova.network.neutron [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Successfully created port: c45bc645-978c-44dd-b928-4568f73165f7 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 639.185526] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e1b07695-51cc-438d-9371-19e04976d723 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.195755] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c49e69f6-6f79-44a6-8018-d66426757e6e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.232958] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bafcec90-9402-4e7d-a6db-86f5570ceb66 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.243355] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2dbaafa0-8316-4d42-a9f1-55f702e8b7e4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.258849] env[62499]: DEBUG nova.compute.provider_tree [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 639.274322] env[62499]: DEBUG oslo_concurrency.lockutils [req-8a377021-722b-4e8e-81c5-f64da9f6059d req-c123d3e0-faf6-45f6-a110-ffd9e9ec8e6c service nova] Releasing lock "refresh_cache-6680c707-aa59-4e00-a2b1-38d465cc6497" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 639.274957] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquired lock "refresh_cache-6680c707-aa59-4e00-a2b1-38d465cc6497" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 639.274957] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 639.762696] env[62499]: DEBUG nova.scheduler.client.report [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 639.775333] env[62499]: DEBUG nova.compute.manager [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 639.801308] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 639.821500] env[62499]: DEBUG nova.virt.hardware [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 639.822273] env[62499]: DEBUG nova.virt.hardware [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 639.822499] env[62499]: DEBUG nova.virt.hardware [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 639.823189] env[62499]: DEBUG nova.virt.hardware [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 639.823189] env[62499]: DEBUG nova.virt.hardware [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 639.823189] env[62499]: DEBUG nova.virt.hardware [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 639.823322] env[62499]: DEBUG nova.virt.hardware [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 639.823534] env[62499]: DEBUG nova.virt.hardware [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 639.823731] env[62499]: DEBUG nova.virt.hardware [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 639.825022] env[62499]: DEBUG nova.virt.hardware [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 639.825022] env[62499]: DEBUG nova.virt.hardware [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 639.825022] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7326e878-6e40-40d2-97ac-76974d34c7dd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.841244] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c1c75bd-5522-493c-804d-41732063612f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.848964] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Acquiring lock "f5b10ac7-e20c-439c-bc6f-a3cf2dd26122" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 639.849878] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Lock "f5b10ac7-e20c-439c-bc6f-a3cf2dd26122" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 639.960186] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 640.269845] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.528s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 640.271080] env[62499]: DEBUG nova.compute.manager [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 640.273771] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.565s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 640.329838] env[62499]: DEBUG nova.compute.manager [req-5a7ab611-c348-4fa2-9b35-2fe7a3d47d08 req-4424ceb5-b4c4-4f25-bba6-a6d08b9103e9 service nova] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Received event network-vif-deleted-77d6d7dd-312e-4215-8183-65f7e69ab012 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 640.469044] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Releasing lock "refresh_cache-6680c707-aa59-4e00-a2b1-38d465cc6497" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 640.469044] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 640.469044] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 640.469044] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-12cb7676-5d87-4f89-ae0e-3cd1a151e484 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.478293] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5267f28b-20d3-4824-8560-8cf435877be1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.501654] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6680c707-aa59-4e00-a2b1-38d465cc6497 could not be found. [ 640.501819] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 640.502069] env[62499]: INFO nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Took 0.04 seconds to destroy the instance on the hypervisor. [ 640.502275] env[62499]: DEBUG oslo.service.loopingcall [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 640.502498] env[62499]: DEBUG nova.compute.manager [-] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 640.502590] env[62499]: DEBUG nova.network.neutron [-] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 640.521285] env[62499]: DEBUG nova.network.neutron [-] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 640.748047] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Acquiring lock "ff822880-41dc-429f-80fb-a1ddc0441ea3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 640.748047] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Lock "ff822880-41dc-429f-80fb-a1ddc0441ea3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 640.779696] env[62499]: DEBUG nova.compute.utils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 640.786600] env[62499]: DEBUG nova.compute.manager [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 640.786732] env[62499]: DEBUG nova.network.neutron [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 640.863029] env[62499]: DEBUG nova.policy [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c97a57aba8534d8691106d790547d0d1', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3ebc1a5cf86945f381bc1942d2f83754', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 640.925178] env[62499]: DEBUG nova.compute.manager [req-16e7baf0-fdb1-43a0-bd9c-38294cb053cc req-d41027d8-b522-47e2-b963-a2cb074a75db service nova] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Received event network-changed-c45bc645-978c-44dd-b928-4568f73165f7 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 640.925178] env[62499]: DEBUG nova.compute.manager [req-16e7baf0-fdb1-43a0-bd9c-38294cb053cc req-d41027d8-b522-47e2-b963-a2cb074a75db service nova] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Refreshing instance network info cache due to event network-changed-c45bc645-978c-44dd-b928-4568f73165f7. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 640.925178] env[62499]: DEBUG oslo_concurrency.lockutils [req-16e7baf0-fdb1-43a0-bd9c-38294cb053cc req-d41027d8-b522-47e2-b963-a2cb074a75db service nova] Acquiring lock "refresh_cache-2a963e2c-7ee3-41a4-842b-06c151d27d84" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 640.925178] env[62499]: DEBUG oslo_concurrency.lockutils [req-16e7baf0-fdb1-43a0-bd9c-38294cb053cc req-d41027d8-b522-47e2-b963-a2cb074a75db service nova] Acquired lock "refresh_cache-2a963e2c-7ee3-41a4-842b-06c151d27d84" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 640.925178] env[62499]: DEBUG nova.network.neutron [req-16e7baf0-fdb1-43a0-bd9c-38294cb053cc req-d41027d8-b522-47e2-b963-a2cb074a75db service nova] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Refreshing network info cache for port c45bc645-978c-44dd-b928-4568f73165f7 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 641.023331] env[62499]: DEBUG nova.network.neutron [-] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 641.211151] env[62499]: ERROR nova.compute.manager [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port c45bc645-978c-44dd-b928-4568f73165f7, please check neutron logs for more information. [ 641.211151] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 641.211151] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 641.211151] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 641.211151] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 641.211151] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 641.211151] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 641.211151] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 641.211151] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 641.211151] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 641.211151] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 641.211151] env[62499]: ERROR nova.compute.manager raise self.value [ 641.211151] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 641.211151] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 641.211151] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 641.211151] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 641.212693] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 641.212693] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 641.212693] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port c45bc645-978c-44dd-b928-4568f73165f7, please check neutron logs for more information. [ 641.212693] env[62499]: ERROR nova.compute.manager [ 641.212693] env[62499]: Traceback (most recent call last): [ 641.212693] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 641.212693] env[62499]: listener.cb(fileno) [ 641.212693] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 641.212693] env[62499]: result = function(*args, **kwargs) [ 641.212693] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 641.212693] env[62499]: return func(*args, **kwargs) [ 641.212693] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 641.212693] env[62499]: raise e [ 641.212693] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 641.212693] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 641.212693] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 641.212693] env[62499]: created_port_ids = self._update_ports_for_instance( [ 641.212693] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 641.212693] env[62499]: with excutils.save_and_reraise_exception(): [ 641.212693] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 641.212693] env[62499]: self.force_reraise() [ 641.212693] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 641.212693] env[62499]: raise self.value [ 641.212693] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 641.212693] env[62499]: updated_port = self._update_port( [ 641.212693] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 641.212693] env[62499]: _ensure_no_port_binding_failure(port) [ 641.212693] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 641.212693] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 641.213773] env[62499]: nova.exception.PortBindingFailed: Binding failed for port c45bc645-978c-44dd-b928-4568f73165f7, please check neutron logs for more information. [ 641.213773] env[62499]: Removing descriptor: 17 [ 641.213773] env[62499]: ERROR nova.compute.manager [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port c45bc645-978c-44dd-b928-4568f73165f7, please check neutron logs for more information. [ 641.213773] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Traceback (most recent call last): [ 641.213773] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 641.213773] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] yield resources [ 641.213773] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 641.213773] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] self.driver.spawn(context, instance, image_meta, [ 641.213773] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 641.213773] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] self._vmops.spawn(context, instance, image_meta, injected_files, [ 641.213773] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 641.213773] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] vm_ref = self.build_virtual_machine(instance, [ 641.214144] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 641.214144] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] vif_infos = vmwarevif.get_vif_info(self._session, [ 641.214144] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 641.214144] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] for vif in network_info: [ 641.214144] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 641.214144] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] return self._sync_wrapper(fn, *args, **kwargs) [ 641.214144] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 641.214144] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] self.wait() [ 641.214144] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 641.214144] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] self[:] = self._gt.wait() [ 641.214144] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 641.214144] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] return self._exit_event.wait() [ 641.214144] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 641.214442] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] result = hub.switch() [ 641.214442] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 641.214442] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] return self.greenlet.switch() [ 641.214442] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 641.214442] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] result = function(*args, **kwargs) [ 641.214442] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 641.214442] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] return func(*args, **kwargs) [ 641.214442] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 641.214442] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] raise e [ 641.214442] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 641.214442] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] nwinfo = self.network_api.allocate_for_instance( [ 641.214442] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 641.214442] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] created_port_ids = self._update_ports_for_instance( [ 641.214744] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 641.214744] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] with excutils.save_and_reraise_exception(): [ 641.214744] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 641.214744] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] self.force_reraise() [ 641.214744] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 641.214744] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] raise self.value [ 641.214744] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 641.214744] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] updated_port = self._update_port( [ 641.214744] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 641.214744] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] _ensure_no_port_binding_failure(port) [ 641.214744] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 641.214744] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] raise exception.PortBindingFailed(port_id=port['id']) [ 641.215067] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] nova.exception.PortBindingFailed: Binding failed for port c45bc645-978c-44dd-b928-4568f73165f7, please check neutron logs for more information. [ 641.215067] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] [ 641.215067] env[62499]: INFO nova.compute.manager [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Terminating instance [ 641.217502] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Acquiring lock "refresh_cache-2a963e2c-7ee3-41a4-842b-06c151d27d84" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 641.284990] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2d64e31-29f2-4d90-bf00-94a78c2f9c40 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.288161] env[62499]: DEBUG nova.compute.manager [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 641.301919] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-882a132b-da27-4b86-97f3-0846d6dd2fa6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.346102] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00f20d5d-eaa7-410e-ab74-9de9bd2ead9f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.354739] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38c70b81-572b-4cea-9878-56435275e1bb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.372318] env[62499]: DEBUG nova.compute.provider_tree [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 641.388617] env[62499]: DEBUG nova.network.neutron [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Successfully created port: ef639489-5e4d-4391-8f65-bd9436c56073 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 641.476872] env[62499]: DEBUG nova.network.neutron [req-16e7baf0-fdb1-43a0-bd9c-38294cb053cc req-d41027d8-b522-47e2-b963-a2cb074a75db service nova] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 641.526971] env[62499]: INFO nova.compute.manager [-] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Took 1.02 seconds to deallocate network for instance. [ 641.529538] env[62499]: DEBUG nova.compute.claims [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 641.529765] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 641.863051] env[62499]: DEBUG nova.network.neutron [req-16e7baf0-fdb1-43a0-bd9c-38294cb053cc req-d41027d8-b522-47e2-b963-a2cb074a75db service nova] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 641.874787] env[62499]: DEBUG nova.scheduler.client.report [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 642.299047] env[62499]: DEBUG nova.compute.manager [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 642.324112] env[62499]: DEBUG nova.virt.hardware [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 642.324112] env[62499]: DEBUG nova.virt.hardware [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 642.324112] env[62499]: DEBUG nova.virt.hardware [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 642.324516] env[62499]: DEBUG nova.virt.hardware [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 642.325233] env[62499]: DEBUG nova.virt.hardware [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 642.325634] env[62499]: DEBUG nova.virt.hardware [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 642.325991] env[62499]: DEBUG nova.virt.hardware [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 642.326296] env[62499]: DEBUG nova.virt.hardware [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 642.326717] env[62499]: DEBUG nova.virt.hardware [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 642.327222] env[62499]: DEBUG nova.virt.hardware [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 642.329019] env[62499]: DEBUG nova.virt.hardware [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 642.329019] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-34ba2825-6046-4c45-af28-8dcf74638f39 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 642.338185] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea69cd8b-8023-4753-89aa-a4be4a6ea654 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 642.368345] env[62499]: DEBUG oslo_concurrency.lockutils [req-16e7baf0-fdb1-43a0-bd9c-38294cb053cc req-d41027d8-b522-47e2-b963-a2cb074a75db service nova] Releasing lock "refresh_cache-2a963e2c-7ee3-41a4-842b-06c151d27d84" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 642.368769] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Acquired lock "refresh_cache-2a963e2c-7ee3-41a4-842b-06c151d27d84" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 642.368960] env[62499]: DEBUG nova.network.neutron [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 642.381325] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.107s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 642.381945] env[62499]: ERROR nova.compute.manager [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port c853a44f-5ddf-4d9b-b501-5cf4d5772cd4, please check neutron logs for more information. [ 642.381945] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Traceback (most recent call last): [ 642.381945] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 642.381945] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] self.driver.spawn(context, instance, image_meta, [ 642.381945] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 642.381945] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 642.381945] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 642.381945] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] vm_ref = self.build_virtual_machine(instance, [ 642.381945] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 642.381945] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] vif_infos = vmwarevif.get_vif_info(self._session, [ 642.381945] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 642.382259] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] for vif in network_info: [ 642.382259] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 642.382259] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] return self._sync_wrapper(fn, *args, **kwargs) [ 642.382259] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 642.382259] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] self.wait() [ 642.382259] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 642.382259] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] self[:] = self._gt.wait() [ 642.382259] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 642.382259] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] return self._exit_event.wait() [ 642.382259] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 642.382259] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] result = hub.switch() [ 642.382259] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 642.382259] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] return self.greenlet.switch() [ 642.382562] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 642.382562] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] result = function(*args, **kwargs) [ 642.382562] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 642.382562] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] return func(*args, **kwargs) [ 642.382562] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 642.382562] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] raise e [ 642.382562] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 642.382562] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] nwinfo = self.network_api.allocate_for_instance( [ 642.382562] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 642.382562] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] created_port_ids = self._update_ports_for_instance( [ 642.382562] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 642.382562] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] with excutils.save_and_reraise_exception(): [ 642.382562] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 642.382880] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] self.force_reraise() [ 642.382880] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 642.382880] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] raise self.value [ 642.382880] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 642.382880] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] updated_port = self._update_port( [ 642.382880] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 642.382880] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] _ensure_no_port_binding_failure(port) [ 642.382880] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 642.382880] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] raise exception.PortBindingFailed(port_id=port['id']) [ 642.382880] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] nova.exception.PortBindingFailed: Binding failed for port c853a44f-5ddf-4d9b-b501-5cf4d5772cd4, please check neutron logs for more information. [ 642.382880] env[62499]: ERROR nova.compute.manager [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] [ 642.383614] env[62499]: DEBUG nova.compute.utils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Binding failed for port c853a44f-5ddf-4d9b-b501-5cf4d5772cd4, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 642.384655] env[62499]: DEBUG oslo_concurrency.lockutils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.063s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 642.385916] env[62499]: INFO nova.compute.claims [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 642.389302] env[62499]: DEBUG nova.compute.manager [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Build of instance 81b1d279-dfad-4a2c-bcfb-043b13d77af6 was re-scheduled: Binding failed for port c853a44f-5ddf-4d9b-b501-5cf4d5772cd4, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 642.389799] env[62499]: DEBUG nova.compute.manager [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 642.390041] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Acquiring lock "refresh_cache-81b1d279-dfad-4a2c-bcfb-043b13d77af6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 642.390221] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Acquired lock "refresh_cache-81b1d279-dfad-4a2c-bcfb-043b13d77af6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 642.390347] env[62499]: DEBUG nova.network.neutron [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 642.899110] env[62499]: DEBUG nova.network.neutron [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 642.934762] env[62499]: DEBUG nova.network.neutron [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 643.069551] env[62499]: DEBUG nova.network.neutron [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 643.082227] env[62499]: ERROR nova.compute.manager [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ef639489-5e4d-4391-8f65-bd9436c56073, please check neutron logs for more information. [ 643.082227] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 643.082227] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 643.082227] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 643.082227] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 643.082227] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 643.082227] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 643.082227] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 643.082227] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 643.082227] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 643.082227] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 643.082227] env[62499]: ERROR nova.compute.manager raise self.value [ 643.082227] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 643.082227] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 643.082227] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 643.082227] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 643.083018] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 643.083018] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 643.083018] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ef639489-5e4d-4391-8f65-bd9436c56073, please check neutron logs for more information. [ 643.083018] env[62499]: ERROR nova.compute.manager [ 643.083018] env[62499]: Traceback (most recent call last): [ 643.083018] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 643.083018] env[62499]: listener.cb(fileno) [ 643.083018] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 643.083018] env[62499]: result = function(*args, **kwargs) [ 643.083018] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 643.083018] env[62499]: return func(*args, **kwargs) [ 643.083018] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 643.083018] env[62499]: raise e [ 643.083018] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 643.083018] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 643.083018] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 643.083018] env[62499]: created_port_ids = self._update_ports_for_instance( [ 643.083018] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 643.083018] env[62499]: with excutils.save_and_reraise_exception(): [ 643.083018] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 643.083018] env[62499]: self.force_reraise() [ 643.083018] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 643.083018] env[62499]: raise self.value [ 643.083018] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 643.083018] env[62499]: updated_port = self._update_port( [ 643.083018] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 643.083018] env[62499]: _ensure_no_port_binding_failure(port) [ 643.083018] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 643.083018] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 643.083793] env[62499]: nova.exception.PortBindingFailed: Binding failed for port ef639489-5e4d-4391-8f65-bd9436c56073, please check neutron logs for more information. [ 643.083793] env[62499]: Removing descriptor: 15 [ 643.083793] env[62499]: ERROR nova.compute.manager [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ef639489-5e4d-4391-8f65-bd9436c56073, please check neutron logs for more information. [ 643.083793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Traceback (most recent call last): [ 643.083793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 643.083793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] yield resources [ 643.083793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 643.083793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] self.driver.spawn(context, instance, image_meta, [ 643.083793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 643.083793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] self._vmops.spawn(context, instance, image_meta, injected_files, [ 643.083793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 643.083793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] vm_ref = self.build_virtual_machine(instance, [ 643.084205] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 643.084205] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] vif_infos = vmwarevif.get_vif_info(self._session, [ 643.084205] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 643.084205] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] for vif in network_info: [ 643.084205] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 643.084205] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] return self._sync_wrapper(fn, *args, **kwargs) [ 643.084205] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 643.084205] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] self.wait() [ 643.084205] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 643.084205] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] self[:] = self._gt.wait() [ 643.084205] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 643.084205] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] return self._exit_event.wait() [ 643.084205] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 643.084624] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] result = hub.switch() [ 643.084624] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 643.084624] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] return self.greenlet.switch() [ 643.084624] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 643.084624] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] result = function(*args, **kwargs) [ 643.084624] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 643.084624] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] return func(*args, **kwargs) [ 643.084624] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 643.084624] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] raise e [ 643.084624] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 643.084624] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] nwinfo = self.network_api.allocate_for_instance( [ 643.084624] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 643.084624] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] created_port_ids = self._update_ports_for_instance( [ 643.084958] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 643.084958] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] with excutils.save_and_reraise_exception(): [ 643.084958] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 643.084958] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] self.force_reraise() [ 643.084958] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 643.084958] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] raise self.value [ 643.084958] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 643.084958] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] updated_port = self._update_port( [ 643.084958] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 643.084958] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] _ensure_no_port_binding_failure(port) [ 643.084958] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 643.084958] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] raise exception.PortBindingFailed(port_id=port['id']) [ 643.085412] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] nova.exception.PortBindingFailed: Binding failed for port ef639489-5e4d-4391-8f65-bd9436c56073, please check neutron logs for more information. [ 643.085412] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] [ 643.085412] env[62499]: INFO nova.compute.manager [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Terminating instance [ 643.087091] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Acquiring lock "refresh_cache-5e18d3cc-62bf-4f12-8e89-2e1bcad15d49" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 643.087743] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Acquired lock "refresh_cache-5e18d3cc-62bf-4f12-8e89-2e1bcad15d49" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 643.087743] env[62499]: DEBUG nova.network.neutron [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 643.116706] env[62499]: DEBUG nova.network.neutron [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 643.230870] env[62499]: DEBUG nova.compute.manager [req-ec370820-22ec-4fad-8759-e441725e50d8 req-5d20c6fc-1f55-4828-b339-26e87d767877 service nova] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Received event network-changed-ef639489-5e4d-4391-8f65-bd9436c56073 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 643.230870] env[62499]: DEBUG nova.compute.manager [req-ec370820-22ec-4fad-8759-e441725e50d8 req-5d20c6fc-1f55-4828-b339-26e87d767877 service nova] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Refreshing instance network info cache due to event network-changed-ef639489-5e4d-4391-8f65-bd9436c56073. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 643.230870] env[62499]: DEBUG oslo_concurrency.lockutils [req-ec370820-22ec-4fad-8759-e441725e50d8 req-5d20c6fc-1f55-4828-b339-26e87d767877 service nova] Acquiring lock "refresh_cache-5e18d3cc-62bf-4f12-8e89-2e1bcad15d49" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 643.280814] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquiring lock "0224e4da-3079-477f-96d8-671985e85c39" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 643.280814] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "0224e4da-3079-477f-96d8-671985e85c39" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 643.318656] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquiring lock "477a1892-0316-42ff-8fc8-269239cb2198" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 643.318656] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "477a1892-0316-42ff-8fc8-269239cb2198" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 643.349905] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquiring lock "399b809e-8926-4348-bfaf-3499de57c1b3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 643.350171] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "399b809e-8926-4348-bfaf-3499de57c1b3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 643.362754] env[62499]: DEBUG nova.compute.manager [req-4bd19154-6651-4513-8b0d-add2e22de020 req-43c97853-542e-4461-9eb8-cdf7de2bfb6e service nova] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Received event network-vif-deleted-c45bc645-978c-44dd-b928-4568f73165f7 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 643.581461] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Releasing lock "refresh_cache-2a963e2c-7ee3-41a4-842b-06c151d27d84" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 643.581461] env[62499]: DEBUG nova.compute.manager [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 643.581461] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 643.582429] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3410fe6e-501c-4d6a-a2c1-7d391aa30987 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 643.598195] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ee1cd19-832e-429a-98fc-fb2b001eaeef {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 643.624793] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Releasing lock "refresh_cache-81b1d279-dfad-4a2c-bcfb-043b13d77af6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 643.625024] env[62499]: DEBUG nova.compute.manager [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 643.625223] env[62499]: DEBUG nova.compute.manager [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 643.625405] env[62499]: DEBUG nova.network.neutron [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 643.628409] env[62499]: DEBUG nova.network.neutron [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 643.645044] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 2a963e2c-7ee3-41a4-842b-06c151d27d84 could not be found. [ 643.645044] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 643.645044] env[62499]: INFO nova.compute.manager [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Took 0.06 seconds to destroy the instance on the hypervisor. [ 643.645044] env[62499]: DEBUG oslo.service.loopingcall [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 643.645044] env[62499]: DEBUG nova.compute.manager [-] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 643.645044] env[62499]: DEBUG nova.network.neutron [-] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 643.660229] env[62499]: DEBUG nova.network.neutron [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 643.666428] env[62499]: DEBUG nova.network.neutron [-] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 643.797434] env[62499]: DEBUG nova.network.neutron [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 643.908114] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquiring lock "6e078697-9fc5-4605-8d4a-a47d9331ba9c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 643.908317] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "6e078697-9fc5-4605-8d4a-a47d9331ba9c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 643.970786] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32dc84b3-4c3e-4392-9475-dadd261057f7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 643.978945] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4cf607e8-3365-4dd7-8464-13d781ba724b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 644.010989] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dac61cb6-69ba-4462-abdd-c5bffd511d33 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 644.019013] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d6f5a9b-c939-4a9c-b3a7-de8aa622ce09 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 644.032665] env[62499]: DEBUG nova.compute.provider_tree [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 644.165351] env[62499]: DEBUG nova.network.neutron [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 644.176020] env[62499]: DEBUG nova.network.neutron [-] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 644.303851] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Releasing lock "refresh_cache-5e18d3cc-62bf-4f12-8e89-2e1bcad15d49" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 644.305821] env[62499]: DEBUG nova.compute.manager [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 644.306079] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 644.306386] env[62499]: DEBUG oslo_concurrency.lockutils [req-ec370820-22ec-4fad-8759-e441725e50d8 req-5d20c6fc-1f55-4828-b339-26e87d767877 service nova] Acquired lock "refresh_cache-5e18d3cc-62bf-4f12-8e89-2e1bcad15d49" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 644.306561] env[62499]: DEBUG nova.network.neutron [req-ec370820-22ec-4fad-8759-e441725e50d8 req-5d20c6fc-1f55-4828-b339-26e87d767877 service nova] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Refreshing network info cache for port ef639489-5e4d-4391-8f65-bd9436c56073 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 644.308255] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-a6c1217b-794f-4738-bf54-53304b8c707e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 644.317053] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d8d9cab-cf44-408d-a86c-d2a359ad1fb1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 644.340123] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49 could not be found. [ 644.340388] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 644.340540] env[62499]: INFO nova.compute.manager [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Took 0.03 seconds to destroy the instance on the hypervisor. [ 644.340769] env[62499]: DEBUG oslo.service.loopingcall [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 644.340983] env[62499]: DEBUG nova.compute.manager [-] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 644.341113] env[62499]: DEBUG nova.network.neutron [-] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 644.357065] env[62499]: DEBUG nova.network.neutron [-] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 644.535458] env[62499]: DEBUG nova.scheduler.client.report [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 644.668703] env[62499]: INFO nova.compute.manager [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: 81b1d279-dfad-4a2c-bcfb-043b13d77af6] Took 1.04 seconds to deallocate network for instance. [ 644.680984] env[62499]: INFO nova.compute.manager [-] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Took 1.04 seconds to deallocate network for instance. [ 644.684979] env[62499]: DEBUG nova.compute.claims [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 644.684979] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 644.799993] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Acquiring lock "e7023ea7-9a39-4e5b-90f6-b2499e06ee9d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 644.800250] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Lock "e7023ea7-9a39-4e5b-90f6-b2499e06ee9d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 644.828411] env[62499]: DEBUG nova.network.neutron [req-ec370820-22ec-4fad-8759-e441725e50d8 req-5d20c6fc-1f55-4828-b339-26e87d767877 service nova] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 644.860167] env[62499]: DEBUG nova.network.neutron [-] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 644.925987] env[62499]: DEBUG nova.network.neutron [req-ec370820-22ec-4fad-8759-e441725e50d8 req-5d20c6fc-1f55-4828-b339-26e87d767877 service nova] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 645.043793] env[62499]: DEBUG oslo_concurrency.lockutils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.660s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 645.044383] env[62499]: DEBUG nova.compute.manager [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 645.047052] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.508s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 645.049043] env[62499]: INFO nova.compute.claims [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 645.255570] env[62499]: DEBUG nova.compute.manager [req-2edbdb07-8beb-48ba-b122-37dd7d829784 req-2a581b71-f43b-4360-a431-6f95f63ebcaf service nova] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Received event network-vif-deleted-ef639489-5e4d-4391-8f65-bd9436c56073 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 645.363053] env[62499]: INFO nova.compute.manager [-] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Took 1.02 seconds to deallocate network for instance. [ 645.367983] env[62499]: DEBUG nova.compute.claims [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 645.367983] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 645.430044] env[62499]: DEBUG oslo_concurrency.lockutils [req-ec370820-22ec-4fad-8759-e441725e50d8 req-5d20c6fc-1f55-4828-b339-26e87d767877 service nova] Releasing lock "refresh_cache-5e18d3cc-62bf-4f12-8e89-2e1bcad15d49" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 645.555928] env[62499]: DEBUG nova.compute.utils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 645.559629] env[62499]: DEBUG nova.compute.manager [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 645.559829] env[62499]: DEBUG nova.network.neutron [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 645.615977] env[62499]: DEBUG nova.policy [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c9f0e85cdee84517959179c0a693e265', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '96b5ee8496a84fb09aaaeef5d6f8a5c7', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 645.705496] env[62499]: INFO nova.scheduler.client.report [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Deleted allocations for instance 81b1d279-dfad-4a2c-bcfb-043b13d77af6 [ 645.951936] env[62499]: DEBUG nova.network.neutron [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Successfully created port: adea3ab7-3634-4b13-896b-43473ad8f651 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 646.065757] env[62499]: DEBUG nova.compute.manager [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 646.217720] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc10af78-33a4-4793-8b62-5d350cc01a55 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Lock "81b1d279-dfad-4a2c-bcfb-043b13d77af6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 76.600s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 646.508626] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1540eb56-3d50-4329-b4e7-bd609d2ff405 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 646.516641] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7f60d3e-a47e-443f-906b-feca8a971480 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 646.553210] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e38de8af-8e10-4b72-b74e-5e6280082815 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 646.562080] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c0e3c919-7dc0-453b-8303-596c076aeb5e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 646.580909] env[62499]: DEBUG nova.compute.provider_tree [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 646.723049] env[62499]: DEBUG nova.compute.manager [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 647.026397] env[62499]: ERROR nova.compute.manager [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port adea3ab7-3634-4b13-896b-43473ad8f651, please check neutron logs for more information. [ 647.026397] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 647.026397] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 647.026397] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 647.026397] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 647.026397] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 647.026397] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 647.026397] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 647.026397] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 647.026397] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 647.026397] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 647.026397] env[62499]: ERROR nova.compute.manager raise self.value [ 647.026397] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 647.026397] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 647.026397] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 647.026397] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 647.026857] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 647.026857] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 647.026857] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port adea3ab7-3634-4b13-896b-43473ad8f651, please check neutron logs for more information. [ 647.026857] env[62499]: ERROR nova.compute.manager [ 647.026857] env[62499]: Traceback (most recent call last): [ 647.026857] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 647.026857] env[62499]: listener.cb(fileno) [ 647.026857] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 647.026857] env[62499]: result = function(*args, **kwargs) [ 647.026857] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 647.026857] env[62499]: return func(*args, **kwargs) [ 647.026857] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 647.026857] env[62499]: raise e [ 647.026857] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 647.026857] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 647.026857] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 647.026857] env[62499]: created_port_ids = self._update_ports_for_instance( [ 647.026857] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 647.026857] env[62499]: with excutils.save_and_reraise_exception(): [ 647.026857] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 647.026857] env[62499]: self.force_reraise() [ 647.026857] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 647.026857] env[62499]: raise self.value [ 647.026857] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 647.026857] env[62499]: updated_port = self._update_port( [ 647.026857] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 647.026857] env[62499]: _ensure_no_port_binding_failure(port) [ 647.026857] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 647.026857] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 647.027814] env[62499]: nova.exception.PortBindingFailed: Binding failed for port adea3ab7-3634-4b13-896b-43473ad8f651, please check neutron logs for more information. [ 647.027814] env[62499]: Removing descriptor: 16 [ 647.082557] env[62499]: DEBUG nova.compute.manager [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 647.086595] env[62499]: DEBUG nova.scheduler.client.report [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 647.110892] env[62499]: DEBUG nova.virt.hardware [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 647.111163] env[62499]: DEBUG nova.virt.hardware [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 647.111323] env[62499]: DEBUG nova.virt.hardware [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 647.111504] env[62499]: DEBUG nova.virt.hardware [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 647.111651] env[62499]: DEBUG nova.virt.hardware [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 647.111801] env[62499]: DEBUG nova.virt.hardware [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 647.112014] env[62499]: DEBUG nova.virt.hardware [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 647.112182] env[62499]: DEBUG nova.virt.hardware [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 647.112348] env[62499]: DEBUG nova.virt.hardware [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 647.112514] env[62499]: DEBUG nova.virt.hardware [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 647.112689] env[62499]: DEBUG nova.virt.hardware [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 647.113742] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-764a7f7d-6ee1-4cc5-9971-99fbe8850d79 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 647.122813] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-abae9733-3587-41e8-9b52-e89b0bba9bcf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 647.136028] env[62499]: ERROR nova.compute.manager [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port adea3ab7-3634-4b13-896b-43473ad8f651, please check neutron logs for more information. [ 647.136028] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Traceback (most recent call last): [ 647.136028] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 647.136028] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] yield resources [ 647.136028] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 647.136028] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] self.driver.spawn(context, instance, image_meta, [ 647.136028] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 647.136028] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 647.136028] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 647.136028] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] vm_ref = self.build_virtual_machine(instance, [ 647.136028] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 647.136381] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] vif_infos = vmwarevif.get_vif_info(self._session, [ 647.136381] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 647.136381] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] for vif in network_info: [ 647.136381] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 647.136381] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] return self._sync_wrapper(fn, *args, **kwargs) [ 647.136381] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 647.136381] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] self.wait() [ 647.136381] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 647.136381] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] self[:] = self._gt.wait() [ 647.136381] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 647.136381] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] return self._exit_event.wait() [ 647.136381] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 647.136381] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] current.throw(*self._exc) [ 647.136705] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 647.136705] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] result = function(*args, **kwargs) [ 647.136705] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 647.136705] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] return func(*args, **kwargs) [ 647.136705] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 647.136705] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] raise e [ 647.136705] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 647.136705] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] nwinfo = self.network_api.allocate_for_instance( [ 647.136705] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 647.136705] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] created_port_ids = self._update_ports_for_instance( [ 647.136705] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 647.136705] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] with excutils.save_and_reraise_exception(): [ 647.136705] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 647.137082] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] self.force_reraise() [ 647.137082] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 647.137082] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] raise self.value [ 647.137082] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 647.137082] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] updated_port = self._update_port( [ 647.137082] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 647.137082] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] _ensure_no_port_binding_failure(port) [ 647.137082] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 647.137082] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] raise exception.PortBindingFailed(port_id=port['id']) [ 647.137082] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] nova.exception.PortBindingFailed: Binding failed for port adea3ab7-3634-4b13-896b-43473ad8f651, please check neutron logs for more information. [ 647.137082] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] [ 647.137082] env[62499]: INFO nova.compute.manager [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Terminating instance [ 647.138646] env[62499]: DEBUG oslo_concurrency.lockutils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Acquiring lock "refresh_cache-0464da14-cb61-4f8f-9bd7-16f970aec0e7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 647.138789] env[62499]: DEBUG oslo_concurrency.lockutils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Acquired lock "refresh_cache-0464da14-cb61-4f8f-9bd7-16f970aec0e7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 647.138957] env[62499]: DEBUG nova.network.neutron [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 647.242561] env[62499]: DEBUG oslo_concurrency.lockutils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 647.280554] env[62499]: DEBUG nova.compute.manager [req-0d017391-21b6-4ba8-b3a7-6e4d022472c4 req-e6295591-1aaa-497b-aef2-84ed1c59762b service nova] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Received event network-changed-adea3ab7-3634-4b13-896b-43473ad8f651 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 647.280708] env[62499]: DEBUG nova.compute.manager [req-0d017391-21b6-4ba8-b3a7-6e4d022472c4 req-e6295591-1aaa-497b-aef2-84ed1c59762b service nova] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Refreshing instance network info cache due to event network-changed-adea3ab7-3634-4b13-896b-43473ad8f651. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 647.280922] env[62499]: DEBUG oslo_concurrency.lockutils [req-0d017391-21b6-4ba8-b3a7-6e4d022472c4 req-e6295591-1aaa-497b-aef2-84ed1c59762b service nova] Acquiring lock "refresh_cache-0464da14-cb61-4f8f-9bd7-16f970aec0e7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 647.571928] env[62499]: DEBUG oslo_concurrency.lockutils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Acquiring lock "ea14c163-3afe-4a8d-8370-43e26ba7af37" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 647.571928] env[62499]: DEBUG oslo_concurrency.lockutils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Lock "ea14c163-3afe-4a8d-8370-43e26ba7af37" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 647.590635] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.543s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 647.591506] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.823s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 647.656278] env[62499]: DEBUG nova.network.neutron [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 647.740685] env[62499]: DEBUG nova.network.neutron [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 648.097688] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Acquiring lock "a90b6897-3839-44ad-935c-9aa9f59fb7e9" by "nova.compute.manager.ComputeManager._validate_instance_group_policy.._do_validation" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 648.097934] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Lock "a90b6897-3839-44ad-935c-9aa9f59fb7e9" acquired by "nova.compute.manager.ComputeManager._validate_instance_group_policy.._do_validation" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 648.245453] env[62499]: DEBUG oslo_concurrency.lockutils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Releasing lock "refresh_cache-0464da14-cb61-4f8f-9bd7-16f970aec0e7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 648.245863] env[62499]: DEBUG nova.compute.manager [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 648.247913] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 648.248723] env[62499]: DEBUG oslo_concurrency.lockutils [req-0d017391-21b6-4ba8-b3a7-6e4d022472c4 req-e6295591-1aaa-497b-aef2-84ed1c59762b service nova] Acquired lock "refresh_cache-0464da14-cb61-4f8f-9bd7-16f970aec0e7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 648.248937] env[62499]: DEBUG nova.network.neutron [req-0d017391-21b6-4ba8-b3a7-6e4d022472c4 req-e6295591-1aaa-497b-aef2-84ed1c59762b service nova] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Refreshing network info cache for port adea3ab7-3634-4b13-896b-43473ad8f651 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 648.249964] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-31ad1fdb-5d94-45c1-b138-89a59b13a661 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.261528] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e2134a0-a8e9-4358-8ace-8563971fcfc9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.287421] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 0464da14-cb61-4f8f-9bd7-16f970aec0e7 could not be found. [ 648.287421] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 648.287563] env[62499]: INFO nova.compute.manager [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Took 0.04 seconds to destroy the instance on the hypervisor. [ 648.287785] env[62499]: DEBUG oslo.service.loopingcall [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 648.288425] env[62499]: DEBUG nova.compute.manager [-] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 648.288425] env[62499]: DEBUG nova.network.neutron [-] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 648.309746] env[62499]: DEBUG nova.network.neutron [-] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 648.523831] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a58d0d7-23eb-4323-9728-917e23d6f76f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.533717] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d86d46e-b9c7-45d4-83b0-0ab3fcebc71f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.565863] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c01bba1f-3478-4354-87a3-58f7afda401f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.573439] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e8beb11e-7dc3-42e0-b7ce-af858d3b7a39 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.587095] env[62499]: DEBUG nova.compute.provider_tree [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 648.603379] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Lock "a90b6897-3839-44ad-935c-9aa9f59fb7e9" "released" by "nova.compute.manager.ComputeManager._validate_instance_group_policy.._do_validation" :: held 0.505s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 648.603899] env[62499]: DEBUG nova.compute.manager [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 648.766944] env[62499]: DEBUG nova.network.neutron [req-0d017391-21b6-4ba8-b3a7-6e4d022472c4 req-e6295591-1aaa-497b-aef2-84ed1c59762b service nova] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 648.812607] env[62499]: DEBUG nova.network.neutron [-] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 648.851037] env[62499]: DEBUG nova.network.neutron [req-0d017391-21b6-4ba8-b3a7-6e4d022472c4 req-e6295591-1aaa-497b-aef2-84ed1c59762b service nova] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 649.089714] env[62499]: DEBUG nova.scheduler.client.report [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 649.109631] env[62499]: DEBUG nova.compute.utils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 649.111744] env[62499]: DEBUG nova.compute.manager [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 649.111744] env[62499]: DEBUG nova.network.neutron [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 649.160163] env[62499]: DEBUG nova.policy [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c80d636b389e4e1286595024e5f98d8e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9e8b3e6c662f4075b818d1a01daebd5d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 649.316594] env[62499]: INFO nova.compute.manager [-] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Took 1.03 seconds to deallocate network for instance. [ 649.320942] env[62499]: DEBUG nova.compute.claims [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 649.321166] env[62499]: DEBUG oslo_concurrency.lockutils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 649.353906] env[62499]: DEBUG oslo_concurrency.lockutils [req-0d017391-21b6-4ba8-b3a7-6e4d022472c4 req-e6295591-1aaa-497b-aef2-84ed1c59762b service nova] Releasing lock "refresh_cache-0464da14-cb61-4f8f-9bd7-16f970aec0e7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 649.354207] env[62499]: DEBUG nova.compute.manager [req-0d017391-21b6-4ba8-b3a7-6e4d022472c4 req-e6295591-1aaa-497b-aef2-84ed1c59762b service nova] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Received event network-vif-deleted-adea3ab7-3634-4b13-896b-43473ad8f651 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 649.451301] env[62499]: DEBUG nova.network.neutron [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Successfully created port: ebcbf4d4-0f22-4534-a39b-616a3c1f921b {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 649.595167] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.003s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 649.595658] env[62499]: ERROR nova.compute.manager [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 29b75515-596f-4fac-b874-1afada623805, please check neutron logs for more information. [ 649.595658] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Traceback (most recent call last): [ 649.595658] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 649.595658] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] self.driver.spawn(context, instance, image_meta, [ 649.595658] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 649.595658] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 649.595658] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 649.595658] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] vm_ref = self.build_virtual_machine(instance, [ 649.595658] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 649.595658] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] vif_infos = vmwarevif.get_vif_info(self._session, [ 649.595658] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 649.595968] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] for vif in network_info: [ 649.595968] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 649.595968] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] return self._sync_wrapper(fn, *args, **kwargs) [ 649.595968] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 649.595968] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] self.wait() [ 649.595968] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 649.595968] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] self[:] = self._gt.wait() [ 649.595968] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 649.595968] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] return self._exit_event.wait() [ 649.595968] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 649.595968] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] result = hub.switch() [ 649.595968] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 649.595968] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] return self.greenlet.switch() [ 649.596295] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 649.596295] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] result = function(*args, **kwargs) [ 649.596295] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 649.596295] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] return func(*args, **kwargs) [ 649.596295] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 649.596295] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] raise e [ 649.596295] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 649.596295] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] nwinfo = self.network_api.allocate_for_instance( [ 649.596295] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 649.596295] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] created_port_ids = self._update_ports_for_instance( [ 649.596295] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 649.596295] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] with excutils.save_and_reraise_exception(): [ 649.596295] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 649.597282] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] self.force_reraise() [ 649.597282] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 649.597282] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] raise self.value [ 649.597282] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 649.597282] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] updated_port = self._update_port( [ 649.597282] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 649.597282] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] _ensure_no_port_binding_failure(port) [ 649.597282] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 649.597282] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] raise exception.PortBindingFailed(port_id=port['id']) [ 649.597282] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] nova.exception.PortBindingFailed: Binding failed for port 29b75515-596f-4fac-b874-1afada623805, please check neutron logs for more information. [ 649.597282] env[62499]: ERROR nova.compute.manager [instance: ccce765d-8df9-4837-b166-71e41bea6de7] [ 649.598244] env[62499]: DEBUG nova.compute.utils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Binding failed for port 29b75515-596f-4fac-b874-1afada623805, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 649.598244] env[62499]: DEBUG oslo_concurrency.lockutils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.595s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 649.599707] env[62499]: INFO nova.compute.claims [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 649.606469] env[62499]: DEBUG nova.compute.manager [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Build of instance ccce765d-8df9-4837-b166-71e41bea6de7 was re-scheduled: Binding failed for port 29b75515-596f-4fac-b874-1afada623805, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 649.606469] env[62499]: DEBUG nova.compute.manager [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 649.606469] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Acquiring lock "refresh_cache-ccce765d-8df9-4837-b166-71e41bea6de7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 649.606469] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Acquired lock "refresh_cache-ccce765d-8df9-4837-b166-71e41bea6de7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 649.606769] env[62499]: DEBUG nova.network.neutron [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 649.611989] env[62499]: DEBUG nova.compute.manager [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 650.128122] env[62499]: DEBUG nova.network.neutron [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 650.182596] env[62499]: DEBUG nova.network.neutron [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 650.243860] env[62499]: DEBUG nova.compute.manager [req-740df451-1a39-4a29-b9c7-d95e0e245cb5 req-0b544357-48f1-4784-a62b-8bdc67dd98de service nova] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Received event network-changed-ebcbf4d4-0f22-4534-a39b-616a3c1f921b {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 650.244070] env[62499]: DEBUG nova.compute.manager [req-740df451-1a39-4a29-b9c7-d95e0e245cb5 req-0b544357-48f1-4784-a62b-8bdc67dd98de service nova] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Refreshing instance network info cache due to event network-changed-ebcbf4d4-0f22-4534-a39b-616a3c1f921b. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 650.244286] env[62499]: DEBUG oslo_concurrency.lockutils [req-740df451-1a39-4a29-b9c7-d95e0e245cb5 req-0b544357-48f1-4784-a62b-8bdc67dd98de service nova] Acquiring lock "refresh_cache-c1cd40be-a66c-4ed9-b329-6d94509b19cf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 650.244426] env[62499]: DEBUG oslo_concurrency.lockutils [req-740df451-1a39-4a29-b9c7-d95e0e245cb5 req-0b544357-48f1-4784-a62b-8bdc67dd98de service nova] Acquired lock "refresh_cache-c1cd40be-a66c-4ed9-b329-6d94509b19cf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 650.244581] env[62499]: DEBUG nova.network.neutron [req-740df451-1a39-4a29-b9c7-d95e0e245cb5 req-0b544357-48f1-4784-a62b-8bdc67dd98de service nova] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Refreshing network info cache for port ebcbf4d4-0f22-4534-a39b-616a3c1f921b {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 650.450787] env[62499]: ERROR nova.compute.manager [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ebcbf4d4-0f22-4534-a39b-616a3c1f921b, please check neutron logs for more information. [ 650.450787] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 650.450787] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 650.450787] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 650.450787] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 650.450787] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 650.450787] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 650.450787] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 650.450787] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 650.450787] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 650.450787] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 650.450787] env[62499]: ERROR nova.compute.manager raise self.value [ 650.450787] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 650.450787] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 650.450787] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 650.450787] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 650.451415] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 650.451415] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 650.451415] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ebcbf4d4-0f22-4534-a39b-616a3c1f921b, please check neutron logs for more information. [ 650.451415] env[62499]: ERROR nova.compute.manager [ 650.451415] env[62499]: Traceback (most recent call last): [ 650.451415] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 650.451415] env[62499]: listener.cb(fileno) [ 650.451415] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 650.451415] env[62499]: result = function(*args, **kwargs) [ 650.451415] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 650.451415] env[62499]: return func(*args, **kwargs) [ 650.451415] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 650.451415] env[62499]: raise e [ 650.451415] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 650.451415] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 650.451415] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 650.451415] env[62499]: created_port_ids = self._update_ports_for_instance( [ 650.451415] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 650.451415] env[62499]: with excutils.save_and_reraise_exception(): [ 650.451415] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 650.451415] env[62499]: self.force_reraise() [ 650.451415] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 650.451415] env[62499]: raise self.value [ 650.451415] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 650.451415] env[62499]: updated_port = self._update_port( [ 650.451415] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 650.451415] env[62499]: _ensure_no_port_binding_failure(port) [ 650.451415] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 650.451415] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 650.452610] env[62499]: nova.exception.PortBindingFailed: Binding failed for port ebcbf4d4-0f22-4534-a39b-616a3c1f921b, please check neutron logs for more information. [ 650.452610] env[62499]: Removing descriptor: 16 [ 650.620567] env[62499]: DEBUG nova.compute.manager [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 650.646608] env[62499]: DEBUG nova.virt.hardware [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 650.646900] env[62499]: DEBUG nova.virt.hardware [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 650.647081] env[62499]: DEBUG nova.virt.hardware [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 650.647270] env[62499]: DEBUG nova.virt.hardware [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 650.647418] env[62499]: DEBUG nova.virt.hardware [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 650.647593] env[62499]: DEBUG nova.virt.hardware [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 650.647820] env[62499]: DEBUG nova.virt.hardware [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 650.647982] env[62499]: DEBUG nova.virt.hardware [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 650.648166] env[62499]: DEBUG nova.virt.hardware [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 650.648330] env[62499]: DEBUG nova.virt.hardware [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 650.648526] env[62499]: DEBUG nova.virt.hardware [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 650.649431] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2bb6c643-e94a-4977-ba10-8239c67934e0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.659586] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5bbb7684-cbf7-4237-bfc2-d5fa45ed5285 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.675671] env[62499]: ERROR nova.compute.manager [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ebcbf4d4-0f22-4534-a39b-616a3c1f921b, please check neutron logs for more information. [ 650.675671] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Traceback (most recent call last): [ 650.675671] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 650.675671] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] yield resources [ 650.675671] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 650.675671] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] self.driver.spawn(context, instance, image_meta, [ 650.675671] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 650.675671] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] self._vmops.spawn(context, instance, image_meta, injected_files, [ 650.675671] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 650.675671] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] vm_ref = self.build_virtual_machine(instance, [ 650.675671] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 650.676078] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] vif_infos = vmwarevif.get_vif_info(self._session, [ 650.676078] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 650.676078] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] for vif in network_info: [ 650.676078] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 650.676078] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] return self._sync_wrapper(fn, *args, **kwargs) [ 650.676078] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 650.676078] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] self.wait() [ 650.676078] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 650.676078] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] self[:] = self._gt.wait() [ 650.676078] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 650.676078] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] return self._exit_event.wait() [ 650.676078] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 650.676078] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] current.throw(*self._exc) [ 650.676438] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 650.676438] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] result = function(*args, **kwargs) [ 650.676438] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 650.676438] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] return func(*args, **kwargs) [ 650.676438] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 650.676438] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] raise e [ 650.676438] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 650.676438] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] nwinfo = self.network_api.allocate_for_instance( [ 650.676438] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 650.676438] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] created_port_ids = self._update_ports_for_instance( [ 650.676438] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 650.676438] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] with excutils.save_and_reraise_exception(): [ 650.676438] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 650.676777] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] self.force_reraise() [ 650.676777] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 650.676777] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] raise self.value [ 650.676777] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 650.676777] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] updated_port = self._update_port( [ 650.676777] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 650.676777] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] _ensure_no_port_binding_failure(port) [ 650.676777] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 650.676777] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] raise exception.PortBindingFailed(port_id=port['id']) [ 650.676777] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] nova.exception.PortBindingFailed: Binding failed for port ebcbf4d4-0f22-4534-a39b-616a3c1f921b, please check neutron logs for more information. [ 650.676777] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] [ 650.676777] env[62499]: INFO nova.compute.manager [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Terminating instance [ 650.681517] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Acquiring lock "refresh_cache-c1cd40be-a66c-4ed9-b329-6d94509b19cf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 650.685032] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Releasing lock "refresh_cache-ccce765d-8df9-4837-b166-71e41bea6de7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 650.685219] env[62499]: DEBUG nova.compute.manager [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 650.685388] env[62499]: DEBUG nova.compute.manager [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 650.685555] env[62499]: DEBUG nova.network.neutron [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 650.700834] env[62499]: DEBUG nova.network.neutron [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 650.764940] env[62499]: DEBUG nova.network.neutron [req-740df451-1a39-4a29-b9c7-d95e0e245cb5 req-0b544357-48f1-4784-a62b-8bdc67dd98de service nova] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 650.864148] env[62499]: DEBUG nova.network.neutron [req-740df451-1a39-4a29-b9c7-d95e0e245cb5 req-0b544357-48f1-4784-a62b-8bdc67dd98de service nova] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 651.009514] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6293bb47-1e3f-4659-b24f-5c22c86c62d9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.018061] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86948ae8-9a3e-452c-b2e5-3617935fe2d3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.046839] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1442a91-48d4-438f-91c2-f4539738f600 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.053859] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f6b281da-651e-458f-81c5-f3d0a03f1a8e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.066235] env[62499]: DEBUG nova.compute.provider_tree [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 651.205623] env[62499]: DEBUG nova.network.neutron [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 651.370355] env[62499]: DEBUG oslo_concurrency.lockutils [req-740df451-1a39-4a29-b9c7-d95e0e245cb5 req-0b544357-48f1-4784-a62b-8bdc67dd98de service nova] Releasing lock "refresh_cache-c1cd40be-a66c-4ed9-b329-6d94509b19cf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 651.370802] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Acquired lock "refresh_cache-c1cd40be-a66c-4ed9-b329-6d94509b19cf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 651.371084] env[62499]: DEBUG nova.network.neutron [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 651.569199] env[62499]: DEBUG nova.scheduler.client.report [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 651.707824] env[62499]: INFO nova.compute.manager [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] [instance: ccce765d-8df9-4837-b166-71e41bea6de7] Took 1.02 seconds to deallocate network for instance. [ 651.889978] env[62499]: DEBUG nova.network.neutron [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 651.969711] env[62499]: DEBUG nova.network.neutron [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 652.074981] env[62499]: DEBUG oslo_concurrency.lockutils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.477s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 652.075625] env[62499]: DEBUG nova.compute.manager [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 652.078378] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 16.079s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 652.078378] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 652.078519] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62499) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 652.078808] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.167s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 652.082042] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94510528-f665-4524-b23e-e7ed4e3fccee {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.090150] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c87b5c87-03a0-48a0-be84-67aa26e3537e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.103480] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-57a8001c-1682-48c1-bd68-3b57f0455c62 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.109773] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-baad9f55-b4ed-4efc-8d5f-db116c310b3a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.138588] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181478MB free_disk=130GB free_vcpus=48 pci_devices=None {{(pid=62499) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 652.138747] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 652.272926] env[62499]: DEBUG nova.compute.manager [req-a3f624df-210f-4d12-b6ee-7f6447144881 req-a22de47d-1e28-45cf-8898-5facc29684ec service nova] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Received event network-vif-deleted-ebcbf4d4-0f22-4534-a39b-616a3c1f921b {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 652.472823] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Releasing lock "refresh_cache-c1cd40be-a66c-4ed9-b329-6d94509b19cf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 652.473323] env[62499]: DEBUG nova.compute.manager [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 652.473533] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 652.473816] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b02d7775-d82f-4891-bc2f-19303e63814b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.483448] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40c8d562-84aa-474b-8d96-91e996b8f994 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.505302] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c1cd40be-a66c-4ed9-b329-6d94509b19cf could not be found. [ 652.505525] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 652.505743] env[62499]: INFO nova.compute.manager [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Took 0.03 seconds to destroy the instance on the hypervisor. [ 652.506010] env[62499]: DEBUG oslo.service.loopingcall [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 652.506820] env[62499]: DEBUG nova.compute.manager [-] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 652.507100] env[62499]: DEBUG nova.network.neutron [-] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 652.522121] env[62499]: DEBUG nova.network.neutron [-] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 652.583171] env[62499]: DEBUG nova.compute.utils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 652.584509] env[62499]: DEBUG nova.compute.manager [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 652.584659] env[62499]: DEBUG nova.network.neutron [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 652.633865] env[62499]: DEBUG nova.policy [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '11251536a5af4def955e95158d82fb81', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '18249883469c4514a8300d99f32482a5', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 652.733185] env[62499]: INFO nova.scheduler.client.report [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Deleted allocations for instance ccce765d-8df9-4837-b166-71e41bea6de7 [ 652.951175] env[62499]: DEBUG nova.network.neutron [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Successfully created port: 9951d694-f7eb-4c34-8163-1bdd62fb42a3 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 652.969888] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dfb9fcc1-8003-427b-a84f-262936332034 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.977311] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de1eef8e-ff2a-4624-8beb-dcc5cbaa05a1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.007101] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ad989ca-1c22-45f7-904a-870484fb0532 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.014983] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e1a5be64-a5b4-410b-b54b-2b42a29a43f2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.024708] env[62499]: DEBUG nova.network.neutron [-] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 653.034193] env[62499]: DEBUG nova.compute.provider_tree [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 653.090036] env[62499]: DEBUG nova.compute.manager [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 653.242899] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c65c1bd5-45db-45ef-8992-d75f966ccaf6 tempest-DeleteServersAdminTestJSON-910099737 tempest-DeleteServersAdminTestJSON-910099737-project-member] Lock "ccce765d-8df9-4837-b166-71e41bea6de7" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 82.200s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 653.527172] env[62499]: INFO nova.compute.manager [-] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Took 1.02 seconds to deallocate network for instance. [ 653.529681] env[62499]: DEBUG nova.compute.claims [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 653.529929] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 653.537865] env[62499]: DEBUG nova.scheduler.client.report [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 653.745494] env[62499]: DEBUG nova.compute.manager [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 654.043287] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.964s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 654.043846] env[62499]: ERROR nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port fc688df4-d52b-4751-a8c0-854113d5e094, please check neutron logs for more information. [ 654.043846] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Traceback (most recent call last): [ 654.043846] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 654.043846] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] self.driver.spawn(context, instance, image_meta, [ 654.043846] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 654.043846] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 654.043846] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 654.043846] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] vm_ref = self.build_virtual_machine(instance, [ 654.043846] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 654.043846] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] vif_infos = vmwarevif.get_vif_info(self._session, [ 654.043846] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 654.044229] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] for vif in network_info: [ 654.044229] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 654.044229] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] return self._sync_wrapper(fn, *args, **kwargs) [ 654.044229] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 654.044229] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] self.wait() [ 654.044229] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 654.044229] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] self[:] = self._gt.wait() [ 654.044229] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 654.044229] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] return self._exit_event.wait() [ 654.044229] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 654.044229] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] result = hub.switch() [ 654.044229] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 654.044229] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] return self.greenlet.switch() [ 654.044758] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 654.044758] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] result = function(*args, **kwargs) [ 654.044758] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 654.044758] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] return func(*args, **kwargs) [ 654.044758] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 654.044758] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] raise e [ 654.044758] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 654.044758] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] nwinfo = self.network_api.allocate_for_instance( [ 654.044758] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 654.044758] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] created_port_ids = self._update_ports_for_instance( [ 654.044758] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 654.044758] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] with excutils.save_and_reraise_exception(): [ 654.044758] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 654.045204] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] self.force_reraise() [ 654.045204] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 654.045204] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] raise self.value [ 654.045204] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 654.045204] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] updated_port = self._update_port( [ 654.045204] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 654.045204] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] _ensure_no_port_binding_failure(port) [ 654.045204] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 654.045204] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] raise exception.PortBindingFailed(port_id=port['id']) [ 654.045204] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] nova.exception.PortBindingFailed: Binding failed for port fc688df4-d52b-4751-a8c0-854113d5e094, please check neutron logs for more information. [ 654.045204] env[62499]: ERROR nova.compute.manager [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] [ 654.045509] env[62499]: DEBUG nova.compute.utils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Binding failed for port fc688df4-d52b-4751-a8c0-854113d5e094, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 654.046301] env[62499]: DEBUG oslo_concurrency.lockutils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.484s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 654.047795] env[62499]: INFO nova.compute.claims [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 654.052489] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Build of instance 287c3404-9fc1-4369-b9a4-17da918bf78d was re-scheduled: Binding failed for port fc688df4-d52b-4751-a8c0-854113d5e094, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 654.052854] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 654.053198] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "refresh_cache-287c3404-9fc1-4369-b9a4-17da918bf78d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 654.053399] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquired lock "refresh_cache-287c3404-9fc1-4369-b9a4-17da918bf78d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 654.053541] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 654.098496] env[62499]: DEBUG nova.compute.manager [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 654.126558] env[62499]: ERROR nova.compute.manager [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 9951d694-f7eb-4c34-8163-1bdd62fb42a3, please check neutron logs for more information. [ 654.126558] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 654.126558] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 654.126558] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 654.126558] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 654.126558] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 654.126558] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 654.126558] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 654.126558] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 654.126558] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 654.126558] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 654.126558] env[62499]: ERROR nova.compute.manager raise self.value [ 654.126558] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 654.126558] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 654.126558] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 654.126558] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 654.126991] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 654.126991] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 654.126991] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 9951d694-f7eb-4c34-8163-1bdd62fb42a3, please check neutron logs for more information. [ 654.126991] env[62499]: ERROR nova.compute.manager [ 654.126991] env[62499]: Traceback (most recent call last): [ 654.126991] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 654.126991] env[62499]: listener.cb(fileno) [ 654.126991] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 654.126991] env[62499]: result = function(*args, **kwargs) [ 654.126991] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 654.126991] env[62499]: return func(*args, **kwargs) [ 654.126991] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 654.126991] env[62499]: raise e [ 654.126991] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 654.126991] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 654.126991] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 654.126991] env[62499]: created_port_ids = self._update_ports_for_instance( [ 654.126991] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 654.126991] env[62499]: with excutils.save_and_reraise_exception(): [ 654.126991] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 654.126991] env[62499]: self.force_reraise() [ 654.126991] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 654.126991] env[62499]: raise self.value [ 654.126991] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 654.126991] env[62499]: updated_port = self._update_port( [ 654.126991] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 654.126991] env[62499]: _ensure_no_port_binding_failure(port) [ 654.126991] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 654.126991] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 654.127759] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 9951d694-f7eb-4c34-8163-1bdd62fb42a3, please check neutron logs for more information. [ 654.127759] env[62499]: Removing descriptor: 16 [ 654.129043] env[62499]: DEBUG nova.virt.hardware [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:34:08Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='d1118a28-e498-4697-babe-cbcefc1b90f1',id=36,is_public=True,memory_mb=192,name='tempest-test_resize_flavor_-985146130',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 654.129155] env[62499]: DEBUG nova.virt.hardware [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 654.129301] env[62499]: DEBUG nova.virt.hardware [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 654.129670] env[62499]: DEBUG nova.virt.hardware [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 654.129670] env[62499]: DEBUG nova.virt.hardware [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 654.129775] env[62499]: DEBUG nova.virt.hardware [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 654.129994] env[62499]: DEBUG nova.virt.hardware [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 654.132485] env[62499]: DEBUG nova.virt.hardware [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 654.132485] env[62499]: DEBUG nova.virt.hardware [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 654.132485] env[62499]: DEBUG nova.virt.hardware [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 654.132485] env[62499]: DEBUG nova.virt.hardware [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 654.132485] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-53b3b2d6-48a4-4a8a-b2fa-b31b89697318 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.139243] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c5ab3cbb-3555-44dd-85be-360afe3115eb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.152848] env[62499]: ERROR nova.compute.manager [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 9951d694-f7eb-4c34-8163-1bdd62fb42a3, please check neutron logs for more information. [ 654.152848] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Traceback (most recent call last): [ 654.152848] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 654.152848] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] yield resources [ 654.152848] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 654.152848] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] self.driver.spawn(context, instance, image_meta, [ 654.152848] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 654.152848] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] self._vmops.spawn(context, instance, image_meta, injected_files, [ 654.152848] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 654.152848] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] vm_ref = self.build_virtual_machine(instance, [ 654.152848] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 654.153204] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] vif_infos = vmwarevif.get_vif_info(self._session, [ 654.153204] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 654.153204] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] for vif in network_info: [ 654.153204] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 654.153204] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] return self._sync_wrapper(fn, *args, **kwargs) [ 654.153204] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 654.153204] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] self.wait() [ 654.153204] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 654.153204] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] self[:] = self._gt.wait() [ 654.153204] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 654.153204] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] return self._exit_event.wait() [ 654.153204] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 654.153204] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] current.throw(*self._exc) [ 654.153613] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 654.153613] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] result = function(*args, **kwargs) [ 654.153613] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 654.153613] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] return func(*args, **kwargs) [ 654.153613] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 654.153613] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] raise e [ 654.153613] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 654.153613] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] nwinfo = self.network_api.allocate_for_instance( [ 654.153613] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 654.153613] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] created_port_ids = self._update_ports_for_instance( [ 654.153613] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 654.153613] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] with excutils.save_and_reraise_exception(): [ 654.153613] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 654.153964] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] self.force_reraise() [ 654.153964] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 654.153964] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] raise self.value [ 654.153964] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 654.153964] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] updated_port = self._update_port( [ 654.153964] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 654.153964] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] _ensure_no_port_binding_failure(port) [ 654.153964] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 654.153964] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] raise exception.PortBindingFailed(port_id=port['id']) [ 654.153964] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] nova.exception.PortBindingFailed: Binding failed for port 9951d694-f7eb-4c34-8163-1bdd62fb42a3, please check neutron logs for more information. [ 654.153964] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] [ 654.153964] env[62499]: INFO nova.compute.manager [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Terminating instance [ 654.156395] env[62499]: DEBUG oslo_concurrency.lockutils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Acquiring lock "refresh_cache-e654ab80-4443-4f60-9d51-6730e1907ffe" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 654.156395] env[62499]: DEBUG oslo_concurrency.lockutils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Acquired lock "refresh_cache-e654ab80-4443-4f60-9d51-6730e1907ffe" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 654.156395] env[62499]: DEBUG nova.network.neutron [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 654.267907] env[62499]: DEBUG oslo_concurrency.lockutils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 654.298592] env[62499]: DEBUG nova.compute.manager [req-f4f8f9cf-7020-408d-9149-30a2c718b74d req-69a019ad-0296-4f1a-9cfa-c728fe8a544a service nova] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Received event network-changed-9951d694-f7eb-4c34-8163-1bdd62fb42a3 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 654.298643] env[62499]: DEBUG nova.compute.manager [req-f4f8f9cf-7020-408d-9149-30a2c718b74d req-69a019ad-0296-4f1a-9cfa-c728fe8a544a service nova] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Refreshing instance network info cache due to event network-changed-9951d694-f7eb-4c34-8163-1bdd62fb42a3. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 654.299053] env[62499]: DEBUG oslo_concurrency.lockutils [req-f4f8f9cf-7020-408d-9149-30a2c718b74d req-69a019ad-0296-4f1a-9cfa-c728fe8a544a service nova] Acquiring lock "refresh_cache-e654ab80-4443-4f60-9d51-6730e1907ffe" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 654.581136] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 654.693497] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 654.697796] env[62499]: DEBUG nova.network.neutron [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 654.802670] env[62499]: DEBUG nova.network.neutron [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 655.195993] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Releasing lock "refresh_cache-287c3404-9fc1-4369-b9a4-17da918bf78d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 655.196267] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 655.196449] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 655.196626] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 655.216050] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 655.305050] env[62499]: DEBUG oslo_concurrency.lockutils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Releasing lock "refresh_cache-e654ab80-4443-4f60-9d51-6730e1907ffe" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 655.305470] env[62499]: DEBUG nova.compute.manager [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 655.305660] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 655.305953] env[62499]: DEBUG oslo_concurrency.lockutils [req-f4f8f9cf-7020-408d-9149-30a2c718b74d req-69a019ad-0296-4f1a-9cfa-c728fe8a544a service nova] Acquired lock "refresh_cache-e654ab80-4443-4f60-9d51-6730e1907ffe" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 655.306139] env[62499]: DEBUG nova.network.neutron [req-f4f8f9cf-7020-408d-9149-30a2c718b74d req-69a019ad-0296-4f1a-9cfa-c728fe8a544a service nova] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Refreshing network info cache for port 9951d694-f7eb-4c34-8163-1bdd62fb42a3 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 655.307147] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-970fb6a0-3cbc-4e05-b9f1-16d71c1f1ddc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.318119] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50610a01-4ec4-46c1-a4bd-918244ba9962 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.341366] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance e654ab80-4443-4f60-9d51-6730e1907ffe could not be found. [ 655.341584] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 655.341761] env[62499]: INFO nova.compute.manager [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Took 0.04 seconds to destroy the instance on the hypervisor. [ 655.342010] env[62499]: DEBUG oslo.service.loopingcall [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 655.344206] env[62499]: DEBUG nova.compute.manager [-] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 655.344352] env[62499]: DEBUG nova.network.neutron [-] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 655.358677] env[62499]: DEBUG nova.network.neutron [-] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 655.434920] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f4655dcf-50a6-4d99-b5f8-e202230d170a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.442849] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d926d80b-4273-46b5-9f08-93d22d3f8ca0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.474168] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dbe8d32b-1ceb-4946-b5cf-1a35ef810554 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.482267] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d1be5f4-9d7a-44cf-ac87-87aa45bcdea3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.497302] env[62499]: DEBUG nova.compute.provider_tree [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 655.717959] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 655.825618] env[62499]: DEBUG nova.network.neutron [req-f4f8f9cf-7020-408d-9149-30a2c718b74d req-69a019ad-0296-4f1a-9cfa-c728fe8a544a service nova] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 655.861069] env[62499]: DEBUG nova.network.neutron [-] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 655.886166] env[62499]: DEBUG nova.network.neutron [req-f4f8f9cf-7020-408d-9149-30a2c718b74d req-69a019ad-0296-4f1a-9cfa-c728fe8a544a service nova] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 656.000303] env[62499]: DEBUG nova.scheduler.client.report [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 656.221750] env[62499]: INFO nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 287c3404-9fc1-4369-b9a4-17da918bf78d] Took 1.02 seconds to deallocate network for instance. [ 656.363198] env[62499]: INFO nova.compute.manager [-] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Took 1.02 seconds to deallocate network for instance. [ 656.365807] env[62499]: DEBUG nova.compute.claims [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 656.365985] env[62499]: DEBUG oslo_concurrency.lockutils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 656.388581] env[62499]: DEBUG oslo_concurrency.lockutils [req-f4f8f9cf-7020-408d-9149-30a2c718b74d req-69a019ad-0296-4f1a-9cfa-c728fe8a544a service nova] Releasing lock "refresh_cache-e654ab80-4443-4f60-9d51-6730e1907ffe" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 656.388879] env[62499]: DEBUG nova.compute.manager [req-f4f8f9cf-7020-408d-9149-30a2c718b74d req-69a019ad-0296-4f1a-9cfa-c728fe8a544a service nova] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Received event network-vif-deleted-9951d694-f7eb-4c34-8163-1bdd62fb42a3 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 656.505514] env[62499]: DEBUG oslo_concurrency.lockutils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.459s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 656.506225] env[62499]: DEBUG nova.compute.manager [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 656.508792] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.979s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 657.015130] env[62499]: DEBUG nova.compute.utils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 657.020401] env[62499]: DEBUG nova.compute.manager [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 657.020401] env[62499]: DEBUG nova.network.neutron [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 657.087905] env[62499]: DEBUG nova.policy [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '5b24cb55df0941d88665d71e23a27820', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9334d955af1f4bbf815935fdc0ff3edf', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 657.254271] env[62499]: INFO nova.scheduler.client.report [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Deleted allocations for instance 287c3404-9fc1-4369-b9a4-17da918bf78d [ 657.431400] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97dc6603-d3d2-42ee-b5c1-8fef768249aa {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 657.440775] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-653fcbff-766a-47dd-a7f6-eb025e143125 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 657.471574] env[62499]: DEBUG nova.network.neutron [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Successfully created port: 94469936-9e70-45b5-bbde-a295c9e733a7 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 657.473843] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b4854bf-e2cd-4925-baf6-ad9e2021cd92 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 657.481153] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de631ecb-a5ae-41fa-8d15-4ced27478f6a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 657.493926] env[62499]: DEBUG nova.compute.provider_tree [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 657.522723] env[62499]: DEBUG nova.compute.manager [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 657.765032] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "287c3404-9fc1-4369-b9a4-17da918bf78d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 85.757s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 657.999107] env[62499]: DEBUG nova.scheduler.client.report [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 658.268961] env[62499]: DEBUG nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 658.382233] env[62499]: DEBUG nova.compute.manager [req-e88a0a0e-8962-41a1-b32b-d3b009c6e077 req-ee09f97d-e238-468e-b557-f58a8346ba45 service nova] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Received event network-changed-94469936-9e70-45b5-bbde-a295c9e733a7 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 658.382233] env[62499]: DEBUG nova.compute.manager [req-e88a0a0e-8962-41a1-b32b-d3b009c6e077 req-ee09f97d-e238-468e-b557-f58a8346ba45 service nova] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Refreshing instance network info cache due to event network-changed-94469936-9e70-45b5-bbde-a295c9e733a7. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 658.382233] env[62499]: DEBUG oslo_concurrency.lockutils [req-e88a0a0e-8962-41a1-b32b-d3b009c6e077 req-ee09f97d-e238-468e-b557-f58a8346ba45 service nova] Acquiring lock "refresh_cache-28832fde-2434-4d5a-ae62-550b8c0eb829" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 658.382233] env[62499]: DEBUG oslo_concurrency.lockutils [req-e88a0a0e-8962-41a1-b32b-d3b009c6e077 req-ee09f97d-e238-468e-b557-f58a8346ba45 service nova] Acquired lock "refresh_cache-28832fde-2434-4d5a-ae62-550b8c0eb829" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 658.382233] env[62499]: DEBUG nova.network.neutron [req-e88a0a0e-8962-41a1-b32b-d3b009c6e077 req-ee09f97d-e238-468e-b557-f58a8346ba45 service nova] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Refreshing network info cache for port 94469936-9e70-45b5-bbde-a295c9e733a7 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 658.410959] env[62499]: DEBUG oslo_concurrency.lockutils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Acquiring lock "6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 658.411767] env[62499]: DEBUG oslo_concurrency.lockutils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Lock "6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 658.504924] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.996s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 658.505607] env[62499]: ERROR nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 77d6d7dd-312e-4215-8183-65f7e69ab012, please check neutron logs for more information. [ 658.505607] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Traceback (most recent call last): [ 658.505607] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 658.505607] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] self.driver.spawn(context, instance, image_meta, [ 658.505607] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 658.505607] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] self._vmops.spawn(context, instance, image_meta, injected_files, [ 658.505607] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 658.505607] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] vm_ref = self.build_virtual_machine(instance, [ 658.505607] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 658.505607] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] vif_infos = vmwarevif.get_vif_info(self._session, [ 658.505607] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 658.505980] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] for vif in network_info: [ 658.505980] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 658.505980] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] return self._sync_wrapper(fn, *args, **kwargs) [ 658.505980] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 658.505980] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] self.wait() [ 658.505980] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 658.505980] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] self[:] = self._gt.wait() [ 658.505980] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 658.505980] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] return self._exit_event.wait() [ 658.505980] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 658.505980] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] result = hub.switch() [ 658.505980] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 658.505980] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] return self.greenlet.switch() [ 658.506398] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 658.506398] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] result = function(*args, **kwargs) [ 658.506398] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 658.506398] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] return func(*args, **kwargs) [ 658.506398] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 658.506398] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] raise e [ 658.506398] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 658.506398] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] nwinfo = self.network_api.allocate_for_instance( [ 658.506398] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 658.506398] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] created_port_ids = self._update_ports_for_instance( [ 658.506398] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 658.506398] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] with excutils.save_and_reraise_exception(): [ 658.506398] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 658.506856] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] self.force_reraise() [ 658.506856] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 658.506856] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] raise self.value [ 658.506856] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 658.506856] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] updated_port = self._update_port( [ 658.506856] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 658.506856] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] _ensure_no_port_binding_failure(port) [ 658.506856] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 658.506856] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] raise exception.PortBindingFailed(port_id=port['id']) [ 658.506856] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] nova.exception.PortBindingFailed: Binding failed for port 77d6d7dd-312e-4215-8183-65f7e69ab012, please check neutron logs for more information. [ 658.506856] env[62499]: ERROR nova.compute.manager [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] [ 658.507199] env[62499]: DEBUG nova.compute.utils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Binding failed for port 77d6d7dd-312e-4215-8183-65f7e69ab012, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 658.507823] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.825s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 658.511396] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Build of instance 6680c707-aa59-4e00-a2b1-38d465cc6497 was re-scheduled: Binding failed for port 77d6d7dd-312e-4215-8183-65f7e69ab012, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 658.511977] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 658.512245] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "refresh_cache-6680c707-aa59-4e00-a2b1-38d465cc6497" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 658.512397] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquired lock "refresh_cache-6680c707-aa59-4e00-a2b1-38d465cc6497" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 658.512560] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 658.531460] env[62499]: DEBUG nova.compute.manager [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 658.548766] env[62499]: ERROR nova.compute.manager [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 94469936-9e70-45b5-bbde-a295c9e733a7, please check neutron logs for more information. [ 658.548766] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 658.548766] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 658.548766] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 658.548766] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 658.548766] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 658.548766] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 658.548766] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 658.548766] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 658.548766] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 658.548766] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 658.548766] env[62499]: ERROR nova.compute.manager raise self.value [ 658.548766] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 658.548766] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 658.548766] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 658.548766] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 658.549338] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 658.549338] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 658.549338] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 94469936-9e70-45b5-bbde-a295c9e733a7, please check neutron logs for more information. [ 658.549338] env[62499]: ERROR nova.compute.manager [ 658.549338] env[62499]: Traceback (most recent call last): [ 658.549338] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 658.549338] env[62499]: listener.cb(fileno) [ 658.549338] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 658.549338] env[62499]: result = function(*args, **kwargs) [ 658.549338] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 658.549338] env[62499]: return func(*args, **kwargs) [ 658.549338] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 658.549338] env[62499]: raise e [ 658.549338] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 658.549338] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 658.549338] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 658.549338] env[62499]: created_port_ids = self._update_ports_for_instance( [ 658.549338] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 658.549338] env[62499]: with excutils.save_and_reraise_exception(): [ 658.549338] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 658.549338] env[62499]: self.force_reraise() [ 658.549338] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 658.549338] env[62499]: raise self.value [ 658.549338] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 658.549338] env[62499]: updated_port = self._update_port( [ 658.549338] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 658.549338] env[62499]: _ensure_no_port_binding_failure(port) [ 658.549338] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 658.549338] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 658.550195] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 94469936-9e70-45b5-bbde-a295c9e733a7, please check neutron logs for more information. [ 658.550195] env[62499]: Removing descriptor: 17 [ 658.563948] env[62499]: DEBUG nova.virt.hardware [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 658.564204] env[62499]: DEBUG nova.virt.hardware [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 658.564363] env[62499]: DEBUG nova.virt.hardware [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 658.564538] env[62499]: DEBUG nova.virt.hardware [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 658.564680] env[62499]: DEBUG nova.virt.hardware [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 658.564824] env[62499]: DEBUG nova.virt.hardware [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 658.565074] env[62499]: DEBUG nova.virt.hardware [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 658.565261] env[62499]: DEBUG nova.virt.hardware [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 658.565436] env[62499]: DEBUG nova.virt.hardware [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 658.565595] env[62499]: DEBUG nova.virt.hardware [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 658.565764] env[62499]: DEBUG nova.virt.hardware [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 658.566622] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7666eba6-db98-4e71-9825-d075e98c07e2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 658.576198] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2496f384-259d-4cd2-8309-c8246fb4ecc9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 658.589920] env[62499]: ERROR nova.compute.manager [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 94469936-9e70-45b5-bbde-a295c9e733a7, please check neutron logs for more information. [ 658.589920] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Traceback (most recent call last): [ 658.589920] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 658.589920] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] yield resources [ 658.589920] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 658.589920] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] self.driver.spawn(context, instance, image_meta, [ 658.589920] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 658.589920] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] self._vmops.spawn(context, instance, image_meta, injected_files, [ 658.589920] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 658.589920] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] vm_ref = self.build_virtual_machine(instance, [ 658.589920] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 658.590325] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] vif_infos = vmwarevif.get_vif_info(self._session, [ 658.590325] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 658.590325] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] for vif in network_info: [ 658.590325] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 658.590325] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] return self._sync_wrapper(fn, *args, **kwargs) [ 658.590325] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 658.590325] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] self.wait() [ 658.590325] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 658.590325] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] self[:] = self._gt.wait() [ 658.590325] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 658.590325] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] return self._exit_event.wait() [ 658.590325] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 658.590325] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] current.throw(*self._exc) [ 658.590650] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 658.590650] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] result = function(*args, **kwargs) [ 658.590650] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 658.590650] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] return func(*args, **kwargs) [ 658.590650] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 658.590650] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] raise e [ 658.590650] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 658.590650] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] nwinfo = self.network_api.allocate_for_instance( [ 658.590650] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 658.590650] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] created_port_ids = self._update_ports_for_instance( [ 658.590650] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 658.590650] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] with excutils.save_and_reraise_exception(): [ 658.590650] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 658.591058] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] self.force_reraise() [ 658.591058] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 658.591058] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] raise self.value [ 658.591058] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 658.591058] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] updated_port = self._update_port( [ 658.591058] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 658.591058] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] _ensure_no_port_binding_failure(port) [ 658.591058] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 658.591058] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] raise exception.PortBindingFailed(port_id=port['id']) [ 658.591058] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] nova.exception.PortBindingFailed: Binding failed for port 94469936-9e70-45b5-bbde-a295c9e733a7, please check neutron logs for more information. [ 658.591058] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] [ 658.591058] env[62499]: INFO nova.compute.manager [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Terminating instance [ 658.592460] env[62499]: DEBUG oslo_concurrency.lockutils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Acquiring lock "refresh_cache-28832fde-2434-4d5a-ae62-550b8c0eb829" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 658.793752] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 658.904346] env[62499]: DEBUG nova.network.neutron [req-e88a0a0e-8962-41a1-b32b-d3b009c6e077 req-ee09f97d-e238-468e-b557-f58a8346ba45 service nova] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 658.990243] env[62499]: DEBUG nova.network.neutron [req-e88a0a0e-8962-41a1-b32b-d3b009c6e077 req-ee09f97d-e238-468e-b557-f58a8346ba45 service nova] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 659.033158] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 659.148091] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 659.430184] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6fc7f2b-0662-4c18-be65-f68abbd16af9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 659.438102] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0b48dd5-8bd0-4eed-b4d5-44080889f354 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 659.473273] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1ef3b8b-394d-4fea-8462-b35d40bb0fa3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 659.482184] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0180347e-cbba-44b5-86c8-76d425bc3d93 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 659.496374] env[62499]: DEBUG oslo_concurrency.lockutils [req-e88a0a0e-8962-41a1-b32b-d3b009c6e077 req-ee09f97d-e238-468e-b557-f58a8346ba45 service nova] Releasing lock "refresh_cache-28832fde-2434-4d5a-ae62-550b8c0eb829" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 659.496956] env[62499]: DEBUG nova.compute.provider_tree [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 659.498874] env[62499]: DEBUG oslo_concurrency.lockutils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Acquired lock "refresh_cache-28832fde-2434-4d5a-ae62-550b8c0eb829" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 659.499081] env[62499]: DEBUG nova.network.neutron [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 659.655514] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Releasing lock "refresh_cache-6680c707-aa59-4e00-a2b1-38d465cc6497" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 659.655514] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 659.655514] env[62499]: DEBUG nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 659.655514] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 659.676178] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 659.748633] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "7bdd4371-b5d5-4053-93b1-75d1c5b9835b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 659.748942] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "7bdd4371-b5d5-4053-93b1-75d1c5b9835b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 659.801878] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "63b76c39-11ca-4f1e-b336-2caceece1f72" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 659.802136] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "63b76c39-11ca-4f1e-b336-2caceece1f72" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 660.000469] env[62499]: DEBUG nova.scheduler.client.report [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 660.024018] env[62499]: DEBUG nova.network.neutron [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 660.109553] env[62499]: DEBUG nova.network.neutron [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 660.179795] env[62499]: DEBUG nova.network.neutron [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 660.424644] env[62499]: DEBUG nova.compute.manager [req-9e6db081-d728-4fb6-8f13-8c0946babfd1 req-214360e8-b90b-44f7-b6e0-79830174fdfd service nova] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Received event network-vif-deleted-94469936-9e70-45b5-bbde-a295c9e733a7 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 660.507984] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 660.508675] env[62499]: ERROR nova.compute.manager [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port c45bc645-978c-44dd-b928-4568f73165f7, please check neutron logs for more information. [ 660.508675] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Traceback (most recent call last): [ 660.508675] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 660.508675] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] self.driver.spawn(context, instance, image_meta, [ 660.508675] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 660.508675] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] self._vmops.spawn(context, instance, image_meta, injected_files, [ 660.508675] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 660.508675] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] vm_ref = self.build_virtual_machine(instance, [ 660.508675] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 660.508675] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] vif_infos = vmwarevif.get_vif_info(self._session, [ 660.508675] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 660.509055] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] for vif in network_info: [ 660.509055] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 660.509055] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] return self._sync_wrapper(fn, *args, **kwargs) [ 660.509055] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 660.509055] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] self.wait() [ 660.509055] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 660.509055] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] self[:] = self._gt.wait() [ 660.509055] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 660.509055] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] return self._exit_event.wait() [ 660.509055] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 660.509055] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] result = hub.switch() [ 660.509055] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 660.509055] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] return self.greenlet.switch() [ 660.509449] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 660.509449] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] result = function(*args, **kwargs) [ 660.509449] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 660.509449] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] return func(*args, **kwargs) [ 660.509449] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 660.509449] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] raise e [ 660.509449] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 660.509449] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] nwinfo = self.network_api.allocate_for_instance( [ 660.509449] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 660.509449] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] created_port_ids = self._update_ports_for_instance( [ 660.509449] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 660.509449] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] with excutils.save_and_reraise_exception(): [ 660.509449] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 660.509949] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] self.force_reraise() [ 660.509949] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 660.509949] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] raise self.value [ 660.509949] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 660.509949] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] updated_port = self._update_port( [ 660.509949] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 660.509949] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] _ensure_no_port_binding_failure(port) [ 660.509949] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 660.509949] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] raise exception.PortBindingFailed(port_id=port['id']) [ 660.509949] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] nova.exception.PortBindingFailed: Binding failed for port c45bc645-978c-44dd-b928-4568f73165f7, please check neutron logs for more information. [ 660.509949] env[62499]: ERROR nova.compute.manager [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] [ 660.510283] env[62499]: DEBUG nova.compute.utils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Binding failed for port c45bc645-978c-44dd-b928-4568f73165f7, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 660.510713] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.143s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 660.516206] env[62499]: DEBUG nova.compute.manager [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Build of instance 2a963e2c-7ee3-41a4-842b-06c151d27d84 was re-scheduled: Binding failed for port c45bc645-978c-44dd-b928-4568f73165f7, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 660.516206] env[62499]: DEBUG nova.compute.manager [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 660.516206] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Acquiring lock "refresh_cache-2a963e2c-7ee3-41a4-842b-06c151d27d84" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 660.516206] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Acquired lock "refresh_cache-2a963e2c-7ee3-41a4-842b-06c151d27d84" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 660.516446] env[62499]: DEBUG nova.network.neutron [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 660.612370] env[62499]: DEBUG oslo_concurrency.lockutils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Releasing lock "refresh_cache-28832fde-2434-4d5a-ae62-550b8c0eb829" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 660.612674] env[62499]: DEBUG nova.compute.manager [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 660.612878] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 660.613716] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3dba707a-0aae-48a9-9ec4-987bd30dee89 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 660.622120] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f6fc990-b942-413a-a41e-34426a62ac39 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 660.646015] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 28832fde-2434-4d5a-ae62-550b8c0eb829 could not be found. [ 660.646261] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 660.646960] env[62499]: INFO nova.compute.manager [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Took 0.03 seconds to destroy the instance on the hypervisor. [ 660.646960] env[62499]: DEBUG oslo.service.loopingcall [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 660.647129] env[62499]: DEBUG nova.compute.manager [-] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 660.647129] env[62499]: DEBUG nova.network.neutron [-] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 660.663424] env[62499]: DEBUG nova.network.neutron [-] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 660.682035] env[62499]: INFO nova.compute.manager [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 6680c707-aa59-4e00-a2b1-38d465cc6497] Took 1.03 seconds to deallocate network for instance. [ 661.035204] env[62499]: DEBUG nova.network.neutron [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 661.119374] env[62499]: DEBUG nova.network.neutron [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 661.166261] env[62499]: DEBUG nova.network.neutron [-] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 661.362641] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f46cf6e3-f485-43b5-883b-496d16b17b2f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 661.370625] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8b36bc4-22f9-4768-84fe-c480fd14d22b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 661.399637] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fbc980c4-8512-4ae4-beb5-75b3a832af3b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 661.406918] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8dd4c74b-1f0e-48f2-82de-416ec0c6ec39 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 661.420200] env[62499]: DEBUG nova.compute.provider_tree [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 661.621930] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Releasing lock "refresh_cache-2a963e2c-7ee3-41a4-842b-06c151d27d84" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 661.622236] env[62499]: DEBUG nova.compute.manager [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 661.622386] env[62499]: DEBUG nova.compute.manager [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 661.622553] env[62499]: DEBUG nova.network.neutron [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 661.645618] env[62499]: DEBUG nova.network.neutron [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 661.670571] env[62499]: INFO nova.compute.manager [-] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Took 1.02 seconds to deallocate network for instance. [ 661.673265] env[62499]: DEBUG nova.compute.claims [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 661.673442] env[62499]: DEBUG oslo_concurrency.lockutils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 661.923362] env[62499]: DEBUG nova.scheduler.client.report [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 662.148377] env[62499]: DEBUG nova.network.neutron [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 662.429104] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.918s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 662.429793] env[62499]: ERROR nova.compute.manager [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ef639489-5e4d-4391-8f65-bd9436c56073, please check neutron logs for more information. [ 662.429793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Traceback (most recent call last): [ 662.429793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 662.429793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] self.driver.spawn(context, instance, image_meta, [ 662.429793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 662.429793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] self._vmops.spawn(context, instance, image_meta, injected_files, [ 662.429793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 662.429793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] vm_ref = self.build_virtual_machine(instance, [ 662.429793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 662.429793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] vif_infos = vmwarevif.get_vif_info(self._session, [ 662.429793] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 662.430383] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] for vif in network_info: [ 662.430383] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 662.430383] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] return self._sync_wrapper(fn, *args, **kwargs) [ 662.430383] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 662.430383] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] self.wait() [ 662.430383] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 662.430383] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] self[:] = self._gt.wait() [ 662.430383] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 662.430383] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] return self._exit_event.wait() [ 662.430383] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 662.430383] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] result = hub.switch() [ 662.430383] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 662.430383] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] return self.greenlet.switch() [ 662.431013] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 662.431013] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] result = function(*args, **kwargs) [ 662.431013] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 662.431013] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] return func(*args, **kwargs) [ 662.431013] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 662.431013] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] raise e [ 662.431013] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 662.431013] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] nwinfo = self.network_api.allocate_for_instance( [ 662.431013] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 662.431013] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] created_port_ids = self._update_ports_for_instance( [ 662.431013] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 662.431013] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] with excutils.save_and_reraise_exception(): [ 662.431013] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 662.431621] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] self.force_reraise() [ 662.431621] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 662.431621] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] raise self.value [ 662.431621] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 662.431621] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] updated_port = self._update_port( [ 662.431621] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 662.431621] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] _ensure_no_port_binding_failure(port) [ 662.431621] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 662.431621] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] raise exception.PortBindingFailed(port_id=port['id']) [ 662.431621] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] nova.exception.PortBindingFailed: Binding failed for port ef639489-5e4d-4391-8f65-bd9436c56073, please check neutron logs for more information. [ 662.431621] env[62499]: ERROR nova.compute.manager [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] [ 662.432172] env[62499]: DEBUG nova.compute.utils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Binding failed for port ef639489-5e4d-4391-8f65-bd9436c56073, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 662.432172] env[62499]: DEBUG nova.compute.manager [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Build of instance 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49 was re-scheduled: Binding failed for port ef639489-5e4d-4391-8f65-bd9436c56073, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 662.432610] env[62499]: DEBUG nova.compute.manager [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 662.432779] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Acquiring lock "refresh_cache-5e18d3cc-62bf-4f12-8e89-2e1bcad15d49" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 662.432955] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Acquired lock "refresh_cache-5e18d3cc-62bf-4f12-8e89-2e1bcad15d49" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 662.433158] env[62499]: DEBUG nova.network.neutron [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 662.434105] env[62499]: DEBUG oslo_concurrency.lockutils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.192s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 662.436102] env[62499]: INFO nova.compute.claims [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 662.651400] env[62499]: INFO nova.compute.manager [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] [instance: 2a963e2c-7ee3-41a4-842b-06c151d27d84] Took 1.03 seconds to deallocate network for instance. [ 662.717405] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a6d5335d-2271-419d-aabf-1fbb9fa0c75c tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "6680c707-aa59-4e00-a2b1-38d465cc6497" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 90.675s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 662.718068] env[62499]: Traceback (most recent call last): [ 662.718126] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 662.718126] env[62499]: self.driver.spawn(context, instance, image_meta, [ 662.718126] env[62499]: File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 662.718126] env[62499]: self._vmops.spawn(context, instance, image_meta, injected_files, [ 662.718126] env[62499]: File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 662.718126] env[62499]: vm_ref = self.build_virtual_machine(instance, [ 662.718126] env[62499]: File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 662.718126] env[62499]: vif_infos = vmwarevif.get_vif_info(self._session, [ 662.718126] env[62499]: File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 662.718126] env[62499]: for vif in network_info: [ 662.718126] env[62499]: File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 662.718126] env[62499]: return self._sync_wrapper(fn, *args, **kwargs) [ 662.718126] env[62499]: File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 662.718126] env[62499]: self.wait() [ 662.718126] env[62499]: File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 662.718126] env[62499]: self[:] = self._gt.wait() [ 662.718126] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 662.718126] env[62499]: return self._exit_event.wait() [ 662.718126] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 662.718126] env[62499]: result = hub.switch() [ 662.718126] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 662.718126] env[62499]: return self.greenlet.switch() [ 662.718126] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 662.718126] env[62499]: result = function(*args, **kwargs) [ 662.718763] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 662.718763] env[62499]: return func(*args, **kwargs) [ 662.718763] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 662.718763] env[62499]: raise e [ 662.718763] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 662.718763] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 662.718763] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 662.718763] env[62499]: created_port_ids = self._update_ports_for_instance( [ 662.718763] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 662.718763] env[62499]: with excutils.save_and_reraise_exception(): [ 662.718763] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 662.718763] env[62499]: self.force_reraise() [ 662.718763] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 662.718763] env[62499]: raise self.value [ 662.718763] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 662.718763] env[62499]: updated_port = self._update_port( [ 662.718763] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 662.718763] env[62499]: _ensure_no_port_binding_failure(port) [ 662.718763] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 662.718763] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 662.718763] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 77d6d7dd-312e-4215-8183-65f7e69ab012, please check neutron logs for more information. [ 662.718763] env[62499]: During handling of the above exception, another exception occurred: [ 662.718763] env[62499]: Traceback (most recent call last): [ 662.718763] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2456, in _do_build_and_run_instance [ 662.718763] env[62499]: self._build_and_run_instance(context, instance, image, [ 662.718763] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2748, in _build_and_run_instance [ 662.718763] env[62499]: raise exception.RescheduledException( [ 662.718763] env[62499]: nova.exception.RescheduledException: Build of instance 6680c707-aa59-4e00-a2b1-38d465cc6497 was re-scheduled: Binding failed for port 77d6d7dd-312e-4215-8183-65f7e69ab012, please check neutron logs for more information. [ 662.719824] env[62499]: During handling of the above exception, another exception occurred: [ 662.719824] env[62499]: Traceback (most recent call last): [ 662.719824] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenpool.py", line 87, in _spawn_n_impl [ 662.719824] env[62499]: func(*args, **kwargs) [ 662.719824] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 662.719824] env[62499]: return func(*args, **kwargs) [ 662.719824] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py", line 412, in inner [ 662.719824] env[62499]: return f(*args, **kwargs) [ 662.719824] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2347, in _locked_do_build_and_run_instance [ 662.719824] env[62499]: result = self._do_build_and_run_instance(*args, **kwargs) [ 662.719824] env[62499]: File "/opt/stack/nova/nova/exception_wrapper.py", line 65, in wrapped [ 662.719824] env[62499]: with excutils.save_and_reraise_exception(): [ 662.719824] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 662.719824] env[62499]: self.force_reraise() [ 662.719824] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 662.719824] env[62499]: raise self.value [ 662.719824] env[62499]: File "/opt/stack/nova/nova/exception_wrapper.py", line 63, in wrapped [ 662.719824] env[62499]: return f(self, context, *args, **kw) [ 662.719824] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 166, in decorated_function [ 662.719824] env[62499]: with excutils.save_and_reraise_exception(): [ 662.719824] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 662.719824] env[62499]: self.force_reraise() [ 662.719824] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 662.719824] env[62499]: raise self.value [ 662.719824] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 157, in decorated_function [ 662.719824] env[62499]: return function(self, context, *args, **kwargs) [ 662.719824] env[62499]: File "/opt/stack/nova/nova/compute/utils.py", line 1453, in decorated_function [ 662.719824] env[62499]: return function(self, context, *args, **kwargs) [ 662.719824] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 203, in decorated_function [ 662.719824] env[62499]: return function(self, context, *args, **kwargs) [ 662.719824] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2491, in _do_build_and_run_instance [ 662.719824] env[62499]: instance.save() [ 662.721230] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_versionedobjects/base.py", line 209, in wrapper [ 662.721230] env[62499]: updates, result = self.indirection_api.object_action( [ 662.721230] env[62499]: File "/opt/stack/nova/nova/conductor/rpcapi.py", line 247, in object_action [ 662.721230] env[62499]: return cctxt.call(context, 'object_action', objinst=objinst, [ 662.721230] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/client.py", line 190, in call [ 662.721230] env[62499]: result = self.transport._send( [ 662.721230] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/transport.py", line 123, in _send [ 662.721230] env[62499]: return self._driver.send(target, ctxt, message, [ 662.721230] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 788, in send [ 662.721230] env[62499]: return self._send(target, ctxt, message, wait_for_reply, timeout, [ 662.721230] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 780, in _send [ 662.721230] env[62499]: raise result [ 662.721230] env[62499]: nova.exception_Remote.InstanceNotFound_Remote: Instance 6680c707-aa59-4e00-a2b1-38d465cc6497 could not be found. [ 662.721230] env[62499]: Traceback (most recent call last): [ 662.721230] env[62499]: File "/opt/stack/nova/nova/conductor/manager.py", line 142, in _object_dispatch [ 662.721230] env[62499]: return getattr(target, method)(*args, **kwargs) [ 662.721230] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_versionedobjects/base.py", line 226, in wrapper [ 662.721230] env[62499]: return fn(self, *args, **kwargs) [ 662.721230] env[62499]: File "/opt/stack/nova/nova/objects/instance.py", line 878, in save [ 662.721230] env[62499]: old_ref, inst_ref = db.instance_update_and_get_original( [ 662.721230] env[62499]: File "/opt/stack/nova/nova/db/utils.py", line 35, in wrapper [ 662.721230] env[62499]: return f(*args, **kwargs) [ 662.721230] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/api.py", line 144, in wrapper [ 662.721230] env[62499]: with excutils.save_and_reraise_exception() as ectxt: [ 662.721230] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 662.721230] env[62499]: self.force_reraise() [ 662.721230] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 662.721230] env[62499]: raise self.value [ 662.722421] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/api.py", line 142, in wrapper [ 662.722421] env[62499]: return f(*args, **kwargs) [ 662.722421] env[62499]: File "/opt/stack/nova/nova/db/main/api.py", line 207, in wrapper [ 662.722421] env[62499]: return f(context, *args, **kwargs) [ 662.722421] env[62499]: File "/opt/stack/nova/nova/db/main/api.py", line 2283, in instance_update_and_get_original [ 662.722421] env[62499]: instance_ref = _instance_get_by_uuid(context, instance_uuid, [ 662.722421] env[62499]: File "/opt/stack/nova/nova/db/main/api.py", line 1405, in _instance_get_by_uuid [ 662.722421] env[62499]: raise exception.InstanceNotFound(instance_id=uuid) [ 662.722421] env[62499]: nova.exception.InstanceNotFound: Instance 6680c707-aa59-4e00-a2b1-38d465cc6497 could not be found. [ 662.957117] env[62499]: DEBUG nova.network.neutron [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 663.038639] env[62499]: DEBUG nova.network.neutron [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 663.221127] env[62499]: DEBUG nova.compute.manager [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 663.541474] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Releasing lock "refresh_cache-5e18d3cc-62bf-4f12-8e89-2e1bcad15d49" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 663.541659] env[62499]: DEBUG nova.compute.manager [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 663.541842] env[62499]: DEBUG nova.compute.manager [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 663.542026] env[62499]: DEBUG nova.network.neutron [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 663.561279] env[62499]: DEBUG nova.network.neutron [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 663.678913] env[62499]: INFO nova.scheduler.client.report [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Deleted allocations for instance 2a963e2c-7ee3-41a4-842b-06c151d27d84 [ 663.747101] env[62499]: DEBUG oslo_concurrency.lockutils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 663.909118] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b47e72f-45c1-4478-864d-c722323ca67e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 663.916671] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5ac51b7-46a2-4d3e-b03d-4ca6b715275b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 663.946805] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea4d067e-fdfe-4a44-b982-66f9a2a0eb1a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 663.953937] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76694faa-6129-4e82-92ba-12ec46a0c526 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 663.967295] env[62499]: DEBUG nova.compute.provider_tree [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 664.064792] env[62499]: DEBUG nova.network.neutron [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 664.188441] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dcd70b61-11cd-42ae-9900-5b578a1e7541 tempest-InstanceActionsTestJSON-1270824579 tempest-InstanceActionsTestJSON-1270824579-project-member] Lock "2a963e2c-7ee3-41a4-842b-06c151d27d84" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 88.558s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 664.473049] env[62499]: DEBUG nova.scheduler.client.report [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 664.567964] env[62499]: INFO nova.compute.manager [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] [instance: 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49] Took 1.03 seconds to deallocate network for instance. [ 664.691097] env[62499]: DEBUG nova.compute.manager [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 664.978114] env[62499]: DEBUG oslo_concurrency.lockutils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.544s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 664.978665] env[62499]: DEBUG nova.compute.manager [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 664.981410] env[62499]: DEBUG oslo_concurrency.lockutils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.660s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 665.217460] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 665.492024] env[62499]: DEBUG nova.compute.utils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 665.492024] env[62499]: DEBUG nova.compute.manager [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 665.492024] env[62499]: DEBUG nova.network.neutron [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 665.584302] env[62499]: DEBUG nova.policy [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd0057e858a1e430e9fcb0f49da3935b3', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'b1e75334f9c84afb9adbcdd22ce9834a', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 665.609497] env[62499]: INFO nova.scheduler.client.report [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Deleted allocations for instance 5e18d3cc-62bf-4f12-8e89-2e1bcad15d49 [ 665.939326] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21ed9a72-4ce7-479d-8b2a-657b06210618 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 665.947874] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff623a97-a70b-4c6e-bd63-5d6074ad25be {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 665.983234] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1846d019-5fb5-40c8-a113-e17973d24d2c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 665.994100] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45ccc53d-0513-4135-915c-e26abe470141 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 665.999431] env[62499]: DEBUG nova.compute.manager [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 666.012675] env[62499]: DEBUG nova.compute.provider_tree [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 666.118144] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6647cacc-c07f-470e-9ff7-8dcdbcf14705 tempest-ServerActionsTestOtherB-1271781834 tempest-ServerActionsTestOtherB-1271781834-project-member] Lock "5e18d3cc-62bf-4f12-8e89-2e1bcad15d49" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 87.975s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 666.162513] env[62499]: DEBUG nova.network.neutron [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Successfully created port: 1dd261e2-15db-4d61-8991-84f77bee309f {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 666.518579] env[62499]: DEBUG nova.scheduler.client.report [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 666.621760] env[62499]: DEBUG nova.compute.manager [None req-d57fe4e9-5fe7-4a93-a464-e4d5ae092655 tempest-ServersListShow296Test-1004933319 tempest-ServersListShow296Test-1004933319-project-member] [instance: da2b873c-0cac-4a7d-8639-b13e0f31b7b6] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 667.012665] env[62499]: DEBUG nova.compute.manager [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 667.023435] env[62499]: DEBUG oslo_concurrency.lockutils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.042s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 667.024092] env[62499]: ERROR nova.compute.manager [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port adea3ab7-3634-4b13-896b-43473ad8f651, please check neutron logs for more information. [ 667.024092] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Traceback (most recent call last): [ 667.024092] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 667.024092] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] self.driver.spawn(context, instance, image_meta, [ 667.024092] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 667.024092] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 667.024092] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 667.024092] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] vm_ref = self.build_virtual_machine(instance, [ 667.024092] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 667.024092] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] vif_infos = vmwarevif.get_vif_info(self._session, [ 667.024092] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 667.024417] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] for vif in network_info: [ 667.024417] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 667.024417] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] return self._sync_wrapper(fn, *args, **kwargs) [ 667.024417] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 667.024417] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] self.wait() [ 667.024417] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 667.024417] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] self[:] = self._gt.wait() [ 667.024417] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 667.024417] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] return self._exit_event.wait() [ 667.024417] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 667.024417] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] current.throw(*self._exc) [ 667.024417] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 667.024417] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] result = function(*args, **kwargs) [ 667.024734] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 667.024734] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] return func(*args, **kwargs) [ 667.024734] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 667.024734] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] raise e [ 667.024734] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 667.024734] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] nwinfo = self.network_api.allocate_for_instance( [ 667.024734] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 667.024734] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] created_port_ids = self._update_ports_for_instance( [ 667.024734] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 667.024734] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] with excutils.save_and_reraise_exception(): [ 667.024734] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 667.024734] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] self.force_reraise() [ 667.024734] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 667.025063] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] raise self.value [ 667.025063] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 667.025063] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] updated_port = self._update_port( [ 667.025063] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 667.025063] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] _ensure_no_port_binding_failure(port) [ 667.025063] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 667.025063] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] raise exception.PortBindingFailed(port_id=port['id']) [ 667.025063] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] nova.exception.PortBindingFailed: Binding failed for port adea3ab7-3634-4b13-896b-43473ad8f651, please check neutron logs for more information. [ 667.025063] env[62499]: ERROR nova.compute.manager [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] [ 667.025063] env[62499]: DEBUG nova.compute.utils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Binding failed for port adea3ab7-3634-4b13-896b-43473ad8f651, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 667.025912] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 14.887s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 667.029967] env[62499]: DEBUG nova.compute.manager [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Build of instance 0464da14-cb61-4f8f-9bd7-16f970aec0e7 was re-scheduled: Binding failed for port adea3ab7-3634-4b13-896b-43473ad8f651, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 667.030426] env[62499]: DEBUG nova.compute.manager [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 667.030654] env[62499]: DEBUG oslo_concurrency.lockutils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Acquiring lock "refresh_cache-0464da14-cb61-4f8f-9bd7-16f970aec0e7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 667.030797] env[62499]: DEBUG oslo_concurrency.lockutils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Acquired lock "refresh_cache-0464da14-cb61-4f8f-9bd7-16f970aec0e7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 667.030953] env[62499]: DEBUG nova.network.neutron [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 667.052464] env[62499]: DEBUG nova.virt.hardware [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 667.052712] env[62499]: DEBUG nova.virt.hardware [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 667.052864] env[62499]: DEBUG nova.virt.hardware [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 667.053060] env[62499]: DEBUG nova.virt.hardware [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 667.053205] env[62499]: DEBUG nova.virt.hardware [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 667.053347] env[62499]: DEBUG nova.virt.hardware [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 667.053564] env[62499]: DEBUG nova.virt.hardware [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 667.053768] env[62499]: DEBUG nova.virt.hardware [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 667.053880] env[62499]: DEBUG nova.virt.hardware [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 667.054050] env[62499]: DEBUG nova.virt.hardware [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 667.054222] env[62499]: DEBUG nova.virt.hardware [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 667.055343] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e453e4e3-4a68-4a09-b88b-c6e316829e63 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 667.064687] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7805816-2196-421a-84c9-06aa773e799d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 667.127488] env[62499]: DEBUG nova.compute.manager [None req-d57fe4e9-5fe7-4a93-a464-e4d5ae092655 tempest-ServersListShow296Test-1004933319 tempest-ServersListShow296Test-1004933319-project-member] [instance: da2b873c-0cac-4a7d-8639-b13e0f31b7b6] Instance disappeared before build. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 667.254136] env[62499]: DEBUG nova.compute.manager [req-7d0dc988-d8f9-4447-a4aa-afa3a7a61cbe req-48289e5f-7720-49ce-a2ea-5c7350a980d7 service nova] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Received event network-changed-1dd261e2-15db-4d61-8991-84f77bee309f {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 667.254136] env[62499]: DEBUG nova.compute.manager [req-7d0dc988-d8f9-4447-a4aa-afa3a7a61cbe req-48289e5f-7720-49ce-a2ea-5c7350a980d7 service nova] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Refreshing instance network info cache due to event network-changed-1dd261e2-15db-4d61-8991-84f77bee309f. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 667.260447] env[62499]: DEBUG oslo_concurrency.lockutils [req-7d0dc988-d8f9-4447-a4aa-afa3a7a61cbe req-48289e5f-7720-49ce-a2ea-5c7350a980d7 service nova] Acquiring lock "refresh_cache-873731fb-8d97-41e7-830b-b9ae4281a737" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 667.260447] env[62499]: DEBUG oslo_concurrency.lockutils [req-7d0dc988-d8f9-4447-a4aa-afa3a7a61cbe req-48289e5f-7720-49ce-a2ea-5c7350a980d7 service nova] Acquired lock "refresh_cache-873731fb-8d97-41e7-830b-b9ae4281a737" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 667.260447] env[62499]: DEBUG nova.network.neutron [req-7d0dc988-d8f9-4447-a4aa-afa3a7a61cbe req-48289e5f-7720-49ce-a2ea-5c7350a980d7 service nova] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Refreshing network info cache for port 1dd261e2-15db-4d61-8991-84f77bee309f {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 667.382354] env[62499]: ERROR nova.compute.manager [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1dd261e2-15db-4d61-8991-84f77bee309f, please check neutron logs for more information. [ 667.382354] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 667.382354] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 667.382354] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 667.382354] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 667.382354] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 667.382354] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 667.382354] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 667.382354] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 667.382354] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 667.382354] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 667.382354] env[62499]: ERROR nova.compute.manager raise self.value [ 667.382354] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 667.382354] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 667.382354] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 667.382354] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 667.382810] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 667.382810] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 667.382810] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1dd261e2-15db-4d61-8991-84f77bee309f, please check neutron logs for more information. [ 667.382810] env[62499]: ERROR nova.compute.manager [ 667.382810] env[62499]: Traceback (most recent call last): [ 667.382810] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 667.382810] env[62499]: listener.cb(fileno) [ 667.382810] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 667.382810] env[62499]: result = function(*args, **kwargs) [ 667.382810] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 667.382810] env[62499]: return func(*args, **kwargs) [ 667.382810] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 667.382810] env[62499]: raise e [ 667.382810] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 667.382810] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 667.382810] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 667.382810] env[62499]: created_port_ids = self._update_ports_for_instance( [ 667.382810] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 667.382810] env[62499]: with excutils.save_and_reraise_exception(): [ 667.382810] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 667.382810] env[62499]: self.force_reraise() [ 667.382810] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 667.382810] env[62499]: raise self.value [ 667.382810] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 667.382810] env[62499]: updated_port = self._update_port( [ 667.382810] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 667.382810] env[62499]: _ensure_no_port_binding_failure(port) [ 667.382810] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 667.382810] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 667.383749] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 1dd261e2-15db-4d61-8991-84f77bee309f, please check neutron logs for more information. [ 667.383749] env[62499]: Removing descriptor: 17 [ 667.383749] env[62499]: ERROR nova.compute.manager [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1dd261e2-15db-4d61-8991-84f77bee309f, please check neutron logs for more information. [ 667.383749] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Traceback (most recent call last): [ 667.383749] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 667.383749] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] yield resources [ 667.383749] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 667.383749] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] self.driver.spawn(context, instance, image_meta, [ 667.383749] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 667.383749] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] self._vmops.spawn(context, instance, image_meta, injected_files, [ 667.383749] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 667.383749] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] vm_ref = self.build_virtual_machine(instance, [ 667.384065] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 667.384065] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] vif_infos = vmwarevif.get_vif_info(self._session, [ 667.384065] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 667.384065] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] for vif in network_info: [ 667.384065] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 667.384065] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] return self._sync_wrapper(fn, *args, **kwargs) [ 667.384065] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 667.384065] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] self.wait() [ 667.384065] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 667.384065] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] self[:] = self._gt.wait() [ 667.384065] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 667.384065] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] return self._exit_event.wait() [ 667.384065] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 667.384393] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] result = hub.switch() [ 667.384393] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 667.384393] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] return self.greenlet.switch() [ 667.384393] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 667.384393] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] result = function(*args, **kwargs) [ 667.384393] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 667.384393] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] return func(*args, **kwargs) [ 667.384393] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 667.384393] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] raise e [ 667.384393] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 667.384393] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] nwinfo = self.network_api.allocate_for_instance( [ 667.384393] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 667.384393] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] created_port_ids = self._update_ports_for_instance( [ 667.384718] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 667.384718] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] with excutils.save_and_reraise_exception(): [ 667.384718] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 667.384718] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] self.force_reraise() [ 667.384718] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 667.384718] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] raise self.value [ 667.384718] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 667.384718] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] updated_port = self._update_port( [ 667.384718] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 667.384718] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] _ensure_no_port_binding_failure(port) [ 667.384718] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 667.384718] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] raise exception.PortBindingFailed(port_id=port['id']) [ 667.385028] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] nova.exception.PortBindingFailed: Binding failed for port 1dd261e2-15db-4d61-8991-84f77bee309f, please check neutron logs for more information. [ 667.385028] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] [ 667.385028] env[62499]: INFO nova.compute.manager [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Terminating instance [ 667.386036] env[62499]: DEBUG oslo_concurrency.lockutils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Acquiring lock "refresh_cache-873731fb-8d97-41e7-830b-b9ae4281a737" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 667.551681] env[62499]: DEBUG nova.network.neutron [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 667.642701] env[62499]: DEBUG nova.network.neutron [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 667.647701] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d57fe4e9-5fe7-4a93-a464-e4d5ae092655 tempest-ServersListShow296Test-1004933319 tempest-ServersListShow296Test-1004933319-project-member] Lock "da2b873c-0cac-4a7d-8639-b13e0f31b7b6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 64.663s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 667.782989] env[62499]: DEBUG nova.network.neutron [req-7d0dc988-d8f9-4447-a4aa-afa3a7a61cbe req-48289e5f-7720-49ce-a2ea-5c7350a980d7 service nova] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 667.876631] env[62499]: DEBUG nova.network.neutron [req-7d0dc988-d8f9-4447-a4aa-afa3a7a61cbe req-48289e5f-7720-49ce-a2ea-5c7350a980d7 service nova] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 668.150681] env[62499]: DEBUG oslo_concurrency.lockutils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Releasing lock "refresh_cache-0464da14-cb61-4f8f-9bd7-16f970aec0e7" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 668.150934] env[62499]: DEBUG nova.compute.manager [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 668.151124] env[62499]: DEBUG nova.compute.manager [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 668.151300] env[62499]: DEBUG nova.network.neutron [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 668.154882] env[62499]: DEBUG nova.compute.manager [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 668.176828] env[62499]: DEBUG nova.network.neutron [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 668.380357] env[62499]: DEBUG oslo_concurrency.lockutils [req-7d0dc988-d8f9-4447-a4aa-afa3a7a61cbe req-48289e5f-7720-49ce-a2ea-5c7350a980d7 service nova] Releasing lock "refresh_cache-873731fb-8d97-41e7-830b-b9ae4281a737" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 668.380357] env[62499]: DEBUG oslo_concurrency.lockutils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Acquired lock "refresh_cache-873731fb-8d97-41e7-830b-b9ae4281a737" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 668.380357] env[62499]: DEBUG nova.network.neutron [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 668.567973] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 0464da14-cb61-4f8f-9bd7-16f970aec0e7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 668.568185] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance c1cd40be-a66c-4ed9-b329-6d94509b19cf actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 668.568323] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance e654ab80-4443-4f60-9d51-6730e1907ffe actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 668.568444] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 28832fde-2434-4d5a-ae62-550b8c0eb829 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 668.568572] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 873731fb-8d97-41e7-830b-b9ae4281a737 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 668.678236] env[62499]: DEBUG oslo_concurrency.lockutils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 668.678642] env[62499]: DEBUG nova.network.neutron [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 668.864456] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "008b230e-72b8-43ae-826a-b38111c56e76" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 668.865509] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "008b230e-72b8-43ae-826a-b38111c56e76" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 668.897228] env[62499]: DEBUG nova.network.neutron [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 668.994945] env[62499]: DEBUG nova.network.neutron [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 669.072206] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 9ca13604-2b1b-4d47-8a23-8137384a87a6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 669.144353] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "39bc8b3a-e9d9-48e3-a193-5fbdc2454346" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 669.145036] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "39bc8b3a-e9d9-48e3-a193-5fbdc2454346" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 669.181173] env[62499]: INFO nova.compute.manager [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] [instance: 0464da14-cb61-4f8f-9bd7-16f970aec0e7] Took 1.03 seconds to deallocate network for instance. [ 669.296130] env[62499]: DEBUG nova.compute.manager [req-157c7266-6584-4d77-8fcb-f28fb7f0d405 req-2ebeaf90-075f-42fa-9aa8-e9f62dd8d612 service nova] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Received event network-vif-deleted-1dd261e2-15db-4d61-8991-84f77bee309f {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 669.499954] env[62499]: DEBUG oslo_concurrency.lockutils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Releasing lock "refresh_cache-873731fb-8d97-41e7-830b-b9ae4281a737" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 669.500449] env[62499]: DEBUG nova.compute.manager [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 669.500652] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 669.500965] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f05485e2-02bd-4a7e-8e0e-b031db1053c3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 669.512248] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b34935f9-3a87-4c8f-a877-6b812ac82fab {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 669.534777] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 873731fb-8d97-41e7-830b-b9ae4281a737 could not be found. [ 669.535014] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 669.535190] env[62499]: INFO nova.compute.manager [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Took 0.03 seconds to destroy the instance on the hypervisor. [ 669.535431] env[62499]: DEBUG oslo.service.loopingcall [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 669.535652] env[62499]: DEBUG nova.compute.manager [-] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 669.535745] env[62499]: DEBUG nova.network.neutron [-] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 669.553015] env[62499]: DEBUG nova.network.neutron [-] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 669.575356] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 3b5fb42c-8a81-4575-ba8f-424a101d11ae has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 669.916344] env[62499]: DEBUG oslo_concurrency.lockutils [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquiring lock "15f8e04f-8c14-4315-8686-da4db517f7b6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 669.916599] env[62499]: DEBUG oslo_concurrency.lockutils [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Lock "15f8e04f-8c14-4315-8686-da4db517f7b6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 670.052777] env[62499]: DEBUG nova.network.neutron [-] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 670.078415] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 0899249c-880c-48de-b2c3-2c407115463e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 670.233361] env[62499]: INFO nova.scheduler.client.report [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Deleted allocations for instance 0464da14-cb61-4f8f-9bd7-16f970aec0e7 [ 670.555607] env[62499]: INFO nova.compute.manager [-] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Took 1.02 seconds to deallocate network for instance. [ 670.558616] env[62499]: DEBUG nova.compute.claims [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 670.558808] env[62499]: DEBUG oslo_concurrency.lockutils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 670.584817] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 670.745580] env[62499]: DEBUG oslo_concurrency.lockutils [None req-76450f5c-9e2c-4907-924a-246a3ca4ada0 tempest-FloatingIPsAssociationTestJSON-71830957 tempest-FloatingIPsAssociationTestJSON-71830957-project-member] Lock "0464da14-cb61-4f8f-9bd7-16f970aec0e7" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 87.159s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 671.087656] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 3d9ac347-ffcc-4840-8090-811dbd2865ef has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 671.247968] env[62499]: DEBUG nova.compute.manager [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 671.592069] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 3649431e-f856-4a28-9af7-8f101c3fc2d6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 671.779297] env[62499]: DEBUG oslo_concurrency.lockutils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 672.095969] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 71fa08bc-a5c7-459c-b65b-5dacd9537281 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 672.599010] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 6ae11b59-68af-482a-ad65-543a56d7aaa4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 673.102842] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 6d8b7855-567c-44c3-90ee-fc0daec87f46 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 673.607207] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 8a769120-906c-4e7c-ac19-b7895039efc1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 674.110256] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 75b04055-8c7e-4659-9451-ddcd6d39fe2e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 674.375718] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Acquiring lock "57d21761-431b-4a41-86eb-038f8c35d8ef" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 674.376234] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Lock "57d21761-431b-4a41-86eb-038f8c35d8ef" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 674.614630] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 78a83324-36da-457e-a78d-c82d0a722015 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 675.117457] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance dee17e73-22db-48ff-b0b0-ec7bec850c48 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 675.621708] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance f5b10ac7-e20c-439c-bc6f-a3cf2dd26122 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 676.125249] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance ff822880-41dc-429f-80fb-a1ddc0441ea3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 676.627514] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 0224e4da-3079-477f-96d8-671985e85c39 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 677.130627] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 477a1892-0316-42ff-8fc8-269239cb2198 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 677.633819] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 399b809e-8926-4348-bfaf-3499de57c1b3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 678.136472] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 6e078697-9fc5-4605-8d4a-a47d9331ba9c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 678.639639] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance e7023ea7-9a39-4e5b-90f6-b2499e06ee9d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 679.143712] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance ea14c163-3afe-4a8d-8370-43e26ba7af37 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 679.646936] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 680.150209] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 7bdd4371-b5d5-4053-93b1-75d1c5b9835b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 680.654701] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 63b76c39-11ca-4f1e-b336-2caceece1f72 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 680.655154] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Total usable vcpus: 48, total allocated vcpus: 4 {{(pid=62499) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 680.655346] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1280MB phys_disk=200GB used_disk=4GB total_vcpus=48 used_vcpus=4 pci_stats=[] {{(pid=62499) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 681.150330] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50fd5a4a-4976-4aa6-8790-fb21d4f50ccb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.160814] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d2e9c2bd-1f97-43e9-9946-41c1f2c704f3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.210619] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-495a2002-251e-4dfe-96f9-8c0ff5b43652 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.220524] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a634e53-f9e5-4862-b78f-4e386beabb6d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.233877] env[62499]: DEBUG nova.compute.provider_tree [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 681.737833] env[62499]: DEBUG nova.scheduler.client.report [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 682.244289] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62499) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 682.244289] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 15.218s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 682.244502] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 28.715s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 683.115313] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e21a9d82-4663-4f7a-81b5-f3c611ec3907 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 683.122932] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-06af6d1c-e1b3-4105-b3d0-1bf6dbb3ea0a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 683.153909] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f41c7f1-4cee-43fa-a734-b8be9fac51ef {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 683.161142] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-41b8b742-5da0-4c02-a27c-8f3a5bc95450 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 683.173977] env[62499]: DEBUG nova.compute.provider_tree [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 683.676981] env[62499]: DEBUG nova.scheduler.client.report [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 684.182467] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.938s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 684.183113] env[62499]: ERROR nova.compute.manager [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ebcbf4d4-0f22-4534-a39b-616a3c1f921b, please check neutron logs for more information. [ 684.183113] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Traceback (most recent call last): [ 684.183113] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 684.183113] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] self.driver.spawn(context, instance, image_meta, [ 684.183113] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 684.183113] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] self._vmops.spawn(context, instance, image_meta, injected_files, [ 684.183113] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 684.183113] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] vm_ref = self.build_virtual_machine(instance, [ 684.183113] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 684.183113] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] vif_infos = vmwarevif.get_vif_info(self._session, [ 684.183113] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 684.183495] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] for vif in network_info: [ 684.183495] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 684.183495] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] return self._sync_wrapper(fn, *args, **kwargs) [ 684.183495] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 684.183495] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] self.wait() [ 684.183495] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 684.183495] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] self[:] = self._gt.wait() [ 684.183495] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 684.183495] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] return self._exit_event.wait() [ 684.183495] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 684.183495] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] current.throw(*self._exc) [ 684.183495] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 684.183495] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] result = function(*args, **kwargs) [ 684.183874] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 684.183874] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] return func(*args, **kwargs) [ 684.183874] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 684.183874] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] raise e [ 684.183874] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 684.183874] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] nwinfo = self.network_api.allocate_for_instance( [ 684.183874] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 684.183874] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] created_port_ids = self._update_ports_for_instance( [ 684.183874] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 684.183874] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] with excutils.save_and_reraise_exception(): [ 684.183874] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 684.183874] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] self.force_reraise() [ 684.183874] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 684.184270] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] raise self.value [ 684.184270] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 684.184270] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] updated_port = self._update_port( [ 684.184270] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 684.184270] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] _ensure_no_port_binding_failure(port) [ 684.184270] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 684.184270] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] raise exception.PortBindingFailed(port_id=port['id']) [ 684.184270] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] nova.exception.PortBindingFailed: Binding failed for port ebcbf4d4-0f22-4534-a39b-616a3c1f921b, please check neutron logs for more information. [ 684.184270] env[62499]: ERROR nova.compute.manager [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] [ 684.184270] env[62499]: DEBUG nova.compute.utils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Binding failed for port ebcbf4d4-0f22-4534-a39b-616a3c1f921b, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 684.185030] env[62499]: DEBUG oslo_concurrency.lockutils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 29.917s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 684.186526] env[62499]: INFO nova.compute.claims [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 684.189041] env[62499]: DEBUG nova.compute.manager [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Build of instance c1cd40be-a66c-4ed9-b329-6d94509b19cf was re-scheduled: Binding failed for port ebcbf4d4-0f22-4534-a39b-616a3c1f921b, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 684.189495] env[62499]: DEBUG nova.compute.manager [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 684.189725] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Acquiring lock "refresh_cache-c1cd40be-a66c-4ed9-b329-6d94509b19cf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 684.189871] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Acquired lock "refresh_cache-c1cd40be-a66c-4ed9-b329-6d94509b19cf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 684.190075] env[62499]: DEBUG nova.network.neutron [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 684.709441] env[62499]: DEBUG nova.network.neutron [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 684.822216] env[62499]: DEBUG nova.network.neutron [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 685.323874] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Releasing lock "refresh_cache-c1cd40be-a66c-4ed9-b329-6d94509b19cf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 685.324190] env[62499]: DEBUG nova.compute.manager [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 685.324312] env[62499]: DEBUG nova.compute.manager [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 685.324670] env[62499]: DEBUG nova.network.neutron [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 685.409647] env[62499]: DEBUG nova.network.neutron [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 685.604837] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3699cc2a-f734-4a2f-9abe-cdd58c525ac4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 685.612431] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-104df818-a188-4c66-b693-15302af03871 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 685.641055] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08eddac7-3f7d-4be4-a0c7-1f19003ce87d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 685.647749] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a7692a4-8fc5-413b-9083-6a8766e2d147 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 685.660451] env[62499]: DEBUG nova.compute.provider_tree [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 685.912954] env[62499]: DEBUG nova.network.neutron [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 686.163892] env[62499]: DEBUG nova.scheduler.client.report [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 686.415245] env[62499]: INFO nova.compute.manager [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] [instance: c1cd40be-a66c-4ed9-b329-6d94509b19cf] Took 1.09 seconds to deallocate network for instance. [ 686.668836] env[62499]: DEBUG oslo_concurrency.lockutils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.484s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 686.669531] env[62499]: DEBUG nova.compute.manager [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 686.672593] env[62499]: DEBUG oslo_concurrency.lockutils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 30.307s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 687.176985] env[62499]: DEBUG nova.compute.utils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 687.182856] env[62499]: DEBUG nova.compute.manager [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 687.183052] env[62499]: DEBUG nova.network.neutron [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 687.224890] env[62499]: DEBUG nova.policy [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'bc4be38552f7413e9ed97d5bd407fdcd', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c3bcaeb2afd44f9a8621fbb43f65a1f7', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 687.451590] env[62499]: INFO nova.scheduler.client.report [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Deleted allocations for instance c1cd40be-a66c-4ed9-b329-6d94509b19cf [ 687.546948] env[62499]: DEBUG nova.network.neutron [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Successfully created port: 6a63349e-54c1-4d95-a74c-8515707dd186 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 687.580985] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3af3753e-8702-40b6-8e9f-adcb9179ff23 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 687.588749] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95d09b5d-3e92-4646-a99b-fdf94eb8d152 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 687.617401] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c5c53fab-1f64-4871-b1c2-c555fceeda6f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 687.624634] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ba0452a-c984-4403-a8ae-e929bdd6e54e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 687.637927] env[62499]: DEBUG nova.compute.provider_tree [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 687.683477] env[62499]: DEBUG nova.compute.manager [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 687.963812] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b104f7df-116a-4149-82cb-de7da5ec818d tempest-ServerGroupTestJSON-422224219 tempest-ServerGroupTestJSON-422224219-project-member] Lock "c1cd40be-a66c-4ed9-b329-6d94509b19cf" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 99.347s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 688.140343] env[62499]: DEBUG nova.scheduler.client.report [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 688.295371] env[62499]: DEBUG nova.compute.manager [req-7aae147c-eb65-4a15-b15e-ca7150935b92 req-4680b27e-daed-4c10-81bb-19704ca6313d service nova] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Received event network-changed-6a63349e-54c1-4d95-a74c-8515707dd186 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 688.295609] env[62499]: DEBUG nova.compute.manager [req-7aae147c-eb65-4a15-b15e-ca7150935b92 req-4680b27e-daed-4c10-81bb-19704ca6313d service nova] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Refreshing instance network info cache due to event network-changed-6a63349e-54c1-4d95-a74c-8515707dd186. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 688.295847] env[62499]: DEBUG oslo_concurrency.lockutils [req-7aae147c-eb65-4a15-b15e-ca7150935b92 req-4680b27e-daed-4c10-81bb-19704ca6313d service nova] Acquiring lock "refresh_cache-9ca13604-2b1b-4d47-8a23-8137384a87a6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 688.295996] env[62499]: DEBUG oslo_concurrency.lockutils [req-7aae147c-eb65-4a15-b15e-ca7150935b92 req-4680b27e-daed-4c10-81bb-19704ca6313d service nova] Acquired lock "refresh_cache-9ca13604-2b1b-4d47-8a23-8137384a87a6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 688.298432] env[62499]: DEBUG nova.network.neutron [req-7aae147c-eb65-4a15-b15e-ca7150935b92 req-4680b27e-daed-4c10-81bb-19704ca6313d service nova] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Refreshing network info cache for port 6a63349e-54c1-4d95-a74c-8515707dd186 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 688.465489] env[62499]: DEBUG nova.compute.manager [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 688.495061] env[62499]: ERROR nova.compute.manager [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 6a63349e-54c1-4d95-a74c-8515707dd186, please check neutron logs for more information. [ 688.495061] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 688.495061] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 688.495061] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 688.495061] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 688.495061] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 688.495061] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 688.495061] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 688.495061] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 688.495061] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 688.495061] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 688.495061] env[62499]: ERROR nova.compute.manager raise self.value [ 688.495061] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 688.495061] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 688.495061] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 688.495061] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 688.495736] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 688.495736] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 688.495736] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 6a63349e-54c1-4d95-a74c-8515707dd186, please check neutron logs for more information. [ 688.495736] env[62499]: ERROR nova.compute.manager [ 688.495736] env[62499]: Traceback (most recent call last): [ 688.495736] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 688.495736] env[62499]: listener.cb(fileno) [ 688.495736] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 688.495736] env[62499]: result = function(*args, **kwargs) [ 688.495736] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 688.495736] env[62499]: return func(*args, **kwargs) [ 688.495736] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 688.495736] env[62499]: raise e [ 688.495736] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 688.495736] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 688.495736] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 688.495736] env[62499]: created_port_ids = self._update_ports_for_instance( [ 688.495736] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 688.495736] env[62499]: with excutils.save_and_reraise_exception(): [ 688.495736] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 688.495736] env[62499]: self.force_reraise() [ 688.495736] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 688.495736] env[62499]: raise self.value [ 688.495736] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 688.495736] env[62499]: updated_port = self._update_port( [ 688.495736] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 688.495736] env[62499]: _ensure_no_port_binding_failure(port) [ 688.495736] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 688.495736] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 688.496697] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 6a63349e-54c1-4d95-a74c-8515707dd186, please check neutron logs for more information. [ 688.496697] env[62499]: Removing descriptor: 17 [ 688.647874] env[62499]: DEBUG oslo_concurrency.lockutils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.975s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 688.648990] env[62499]: ERROR nova.compute.manager [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 9951d694-f7eb-4c34-8163-1bdd62fb42a3, please check neutron logs for more information. [ 688.648990] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Traceback (most recent call last): [ 688.648990] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 688.648990] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] self.driver.spawn(context, instance, image_meta, [ 688.648990] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 688.648990] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] self._vmops.spawn(context, instance, image_meta, injected_files, [ 688.648990] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 688.648990] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] vm_ref = self.build_virtual_machine(instance, [ 688.648990] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 688.648990] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] vif_infos = vmwarevif.get_vif_info(self._session, [ 688.648990] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 688.649468] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] for vif in network_info: [ 688.649468] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 688.649468] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] return self._sync_wrapper(fn, *args, **kwargs) [ 688.649468] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 688.649468] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] self.wait() [ 688.649468] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 688.649468] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] self[:] = self._gt.wait() [ 688.649468] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 688.649468] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] return self._exit_event.wait() [ 688.649468] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 688.649468] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] current.throw(*self._exc) [ 688.649468] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 688.649468] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] result = function(*args, **kwargs) [ 688.649784] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 688.649784] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] return func(*args, **kwargs) [ 688.649784] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 688.649784] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] raise e [ 688.649784] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 688.649784] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] nwinfo = self.network_api.allocate_for_instance( [ 688.649784] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 688.649784] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] created_port_ids = self._update_ports_for_instance( [ 688.649784] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 688.649784] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] with excutils.save_and_reraise_exception(): [ 688.649784] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 688.649784] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] self.force_reraise() [ 688.649784] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 688.650164] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] raise self.value [ 688.650164] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 688.650164] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] updated_port = self._update_port( [ 688.650164] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 688.650164] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] _ensure_no_port_binding_failure(port) [ 688.650164] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 688.650164] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] raise exception.PortBindingFailed(port_id=port['id']) [ 688.650164] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] nova.exception.PortBindingFailed: Binding failed for port 9951d694-f7eb-4c34-8163-1bdd62fb42a3, please check neutron logs for more information. [ 688.650164] env[62499]: ERROR nova.compute.manager [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] [ 688.650164] env[62499]: DEBUG nova.compute.utils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Binding failed for port 9951d694-f7eb-4c34-8163-1bdd62fb42a3, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 688.651396] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 29.858s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 688.654563] env[62499]: INFO nova.compute.claims [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 688.658482] env[62499]: DEBUG nova.compute.manager [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Build of instance e654ab80-4443-4f60-9d51-6730e1907ffe was re-scheduled: Binding failed for port 9951d694-f7eb-4c34-8163-1bdd62fb42a3, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 688.658830] env[62499]: DEBUG nova.compute.manager [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 688.659065] env[62499]: DEBUG oslo_concurrency.lockutils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Acquiring lock "refresh_cache-e654ab80-4443-4f60-9d51-6730e1907ffe" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 688.659245] env[62499]: DEBUG oslo_concurrency.lockutils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Acquired lock "refresh_cache-e654ab80-4443-4f60-9d51-6730e1907ffe" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 688.659412] env[62499]: DEBUG nova.network.neutron [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 688.694812] env[62499]: DEBUG nova.compute.manager [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 688.719762] env[62499]: DEBUG nova.virt.hardware [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 688.719999] env[62499]: DEBUG nova.virt.hardware [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 688.720190] env[62499]: DEBUG nova.virt.hardware [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 688.720388] env[62499]: DEBUG nova.virt.hardware [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 688.720535] env[62499]: DEBUG nova.virt.hardware [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 688.720682] env[62499]: DEBUG nova.virt.hardware [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 688.720887] env[62499]: DEBUG nova.virt.hardware [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 688.721064] env[62499]: DEBUG nova.virt.hardware [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 688.721237] env[62499]: DEBUG nova.virt.hardware [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 688.721408] env[62499]: DEBUG nova.virt.hardware [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 688.722031] env[62499]: DEBUG nova.virt.hardware [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 688.722459] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0619c45b-f581-4ca8-a7e8-e16b1fe4d5ae {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 688.730908] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9046569b-e259-4e9a-9708-2720164d956f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 688.744455] env[62499]: ERROR nova.compute.manager [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 6a63349e-54c1-4d95-a74c-8515707dd186, please check neutron logs for more information. [ 688.744455] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Traceback (most recent call last): [ 688.744455] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 688.744455] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] yield resources [ 688.744455] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 688.744455] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] self.driver.spawn(context, instance, image_meta, [ 688.744455] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 688.744455] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 688.744455] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 688.744455] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] vm_ref = self.build_virtual_machine(instance, [ 688.744455] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 688.744890] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] vif_infos = vmwarevif.get_vif_info(self._session, [ 688.744890] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 688.744890] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] for vif in network_info: [ 688.744890] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 688.744890] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] return self._sync_wrapper(fn, *args, **kwargs) [ 688.744890] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 688.744890] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] self.wait() [ 688.744890] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 688.744890] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] self[:] = self._gt.wait() [ 688.744890] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 688.744890] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] return self._exit_event.wait() [ 688.744890] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 688.744890] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] current.throw(*self._exc) [ 688.745327] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 688.745327] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] result = function(*args, **kwargs) [ 688.745327] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 688.745327] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] return func(*args, **kwargs) [ 688.745327] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 688.745327] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] raise e [ 688.745327] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 688.745327] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] nwinfo = self.network_api.allocate_for_instance( [ 688.745327] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 688.745327] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] created_port_ids = self._update_ports_for_instance( [ 688.745327] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 688.745327] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] with excutils.save_and_reraise_exception(): [ 688.745327] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 688.745716] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] self.force_reraise() [ 688.745716] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 688.745716] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] raise self.value [ 688.745716] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 688.745716] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] updated_port = self._update_port( [ 688.745716] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 688.745716] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] _ensure_no_port_binding_failure(port) [ 688.745716] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 688.745716] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] raise exception.PortBindingFailed(port_id=port['id']) [ 688.745716] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] nova.exception.PortBindingFailed: Binding failed for port 6a63349e-54c1-4d95-a74c-8515707dd186, please check neutron logs for more information. [ 688.745716] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] [ 688.745716] env[62499]: INFO nova.compute.manager [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Terminating instance [ 688.746962] env[62499]: DEBUG oslo_concurrency.lockutils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Acquiring lock "refresh_cache-9ca13604-2b1b-4d47-8a23-8137384a87a6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 688.818164] env[62499]: DEBUG nova.network.neutron [req-7aae147c-eb65-4a15-b15e-ca7150935b92 req-4680b27e-daed-4c10-81bb-19704ca6313d service nova] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 688.931350] env[62499]: DEBUG nova.network.neutron [req-7aae147c-eb65-4a15-b15e-ca7150935b92 req-4680b27e-daed-4c10-81bb-19704ca6313d service nova] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 688.989401] env[62499]: DEBUG oslo_concurrency.lockutils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 689.184709] env[62499]: DEBUG nova.network.neutron [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 689.264165] env[62499]: DEBUG nova.network.neutron [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 689.434482] env[62499]: DEBUG oslo_concurrency.lockutils [req-7aae147c-eb65-4a15-b15e-ca7150935b92 req-4680b27e-daed-4c10-81bb-19704ca6313d service nova] Releasing lock "refresh_cache-9ca13604-2b1b-4d47-8a23-8137384a87a6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 689.434892] env[62499]: DEBUG oslo_concurrency.lockutils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Acquired lock "refresh_cache-9ca13604-2b1b-4d47-8a23-8137384a87a6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 689.435088] env[62499]: DEBUG nova.network.neutron [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 689.766785] env[62499]: DEBUG oslo_concurrency.lockutils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Releasing lock "refresh_cache-e654ab80-4443-4f60-9d51-6730e1907ffe" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 689.767055] env[62499]: DEBUG nova.compute.manager [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 689.767244] env[62499]: DEBUG nova.compute.manager [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 689.767426] env[62499]: DEBUG nova.network.neutron [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 689.792508] env[62499]: DEBUG nova.network.neutron [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 689.961212] env[62499]: DEBUG nova.network.neutron [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 690.089604] env[62499]: DEBUG nova.network.neutron [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 690.095217] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28903414-b12f-4206-8c6d-5491edcad9f9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 690.105445] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de35b450-97b1-4d62-a07d-a2be5d3436c0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 690.138994] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f0a39405-36ad-4c39-9cca-88fad57ccaf0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 690.146746] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d523aef-fc04-46e1-9ac1-b65d234b2497 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 690.160158] env[62499]: DEBUG nova.compute.provider_tree [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 690.299775] env[62499]: DEBUG nova.network.neutron [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 690.330414] env[62499]: DEBUG nova.compute.manager [req-4d5d59ab-bcd9-47d4-828d-4e290734dfa2 req-1e505bf4-ece1-481a-9e7a-ea33e8df58c0 service nova] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Received event network-vif-deleted-6a63349e-54c1-4d95-a74c-8515707dd186 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 690.592517] env[62499]: DEBUG oslo_concurrency.lockutils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Releasing lock "refresh_cache-9ca13604-2b1b-4d47-8a23-8137384a87a6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 690.592949] env[62499]: DEBUG nova.compute.manager [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 690.593183] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 690.593440] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-405795ab-93b5-4da3-966a-24870e57ed39 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 690.602258] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-daafcbd3-6208-46a2-9b12-b27d4ea62702 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 690.624063] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9ca13604-2b1b-4d47-8a23-8137384a87a6 could not be found. [ 690.624300] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 690.624484] env[62499]: INFO nova.compute.manager [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Took 0.03 seconds to destroy the instance on the hypervisor. [ 690.624725] env[62499]: DEBUG oslo.service.loopingcall [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 690.624938] env[62499]: DEBUG nova.compute.manager [-] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 690.625048] env[62499]: DEBUG nova.network.neutron [-] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 690.644590] env[62499]: DEBUG nova.network.neutron [-] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 690.664951] env[62499]: DEBUG nova.scheduler.client.report [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 690.800487] env[62499]: INFO nova.compute.manager [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] [instance: e654ab80-4443-4f60-9d51-6730e1907ffe] Took 1.03 seconds to deallocate network for instance. [ 691.150138] env[62499]: DEBUG nova.network.neutron [-] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 691.169205] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.518s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 691.169611] env[62499]: DEBUG nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 691.172397] env[62499]: DEBUG oslo_concurrency.lockutils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 29.499s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 691.653084] env[62499]: INFO nova.compute.manager [-] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Took 1.03 seconds to deallocate network for instance. [ 691.655791] env[62499]: DEBUG nova.compute.claims [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 691.656011] env[62499]: DEBUG oslo_concurrency.lockutils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 691.677420] env[62499]: DEBUG nova.compute.utils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 691.681907] env[62499]: DEBUG nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 691.681907] env[62499]: DEBUG nova.network.neutron [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 691.730394] env[62499]: DEBUG nova.policy [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c790f5999395473dadd46868c37897d5', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9ca4a4fc017a45cfbd044295ee6864bf', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 691.829029] env[62499]: INFO nova.scheduler.client.report [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Deleted allocations for instance e654ab80-4443-4f60-9d51-6730e1907ffe [ 692.048621] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af1f1df9-0b4a-47d0-8532-f624a6d12fe1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.056520] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ad26c6d-19f4-406a-a753-2f1466f5320d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.088595] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2a258cb-9831-4071-a89e-f20c5945e7c4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.095960] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4bb79118-c32f-4d48-a5c8-495a7c68b519 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.109798] env[62499]: DEBUG nova.compute.provider_tree [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 692.185421] env[62499]: DEBUG nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 692.193143] env[62499]: DEBUG nova.network.neutron [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Successfully created port: 39337e32-ee39-47e5-ad79-389ade158ef2 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 692.339536] env[62499]: DEBUG oslo_concurrency.lockutils [None req-eac4af78-090c-4a1e-bb7b-16a53d00c5ff tempest-MigrationsAdminTest-439471344 tempest-MigrationsAdminTest-439471344-project-member] Lock "e654ab80-4443-4f60-9d51-6730e1907ffe" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 102.458s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 692.613149] env[62499]: DEBUG nova.scheduler.client.report [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 692.691480] env[62499]: INFO nova.virt.block_device [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Booting with volume 7a0a7388-8cdd-491f-9a66-930875cdefd6 at /dev/sda [ 692.734772] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-99dd4a5b-bd89-41a4-9e82-e1ed14b212ce {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.743641] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d26b97f0-7e6a-4516-8263-f4510d20cfb2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.766559] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-6cd098fe-de2e-490a-8a32-ed94c3a6ad0a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.776405] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-852d5497-f125-47f5-b38f-d699ed709eeb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.801883] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2b13bb0d-b571-4aca-94ed-e8927974c89c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.808031] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6749c170-0fc8-409a-bba1-7aee64733f52 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.820687] env[62499]: DEBUG nova.virt.block_device [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Updating existing volume attachment record: aee7de93-3e41-4c78-a15a-72299bf32a2a {{(pid=62499) _volume_attach /opt/stack/nova/nova/virt/block_device.py:666}} [ 692.846011] env[62499]: DEBUG nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 693.050176] env[62499]: DEBUG nova.compute.manager [req-a9e2db98-3cc8-4ab8-9fb1-2d5897f75a73 req-8852b2b7-5e60-4829-a03c-5530dc7194c6 service nova] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Received event network-changed-39337e32-ee39-47e5-ad79-389ade158ef2 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 693.054087] env[62499]: DEBUG nova.compute.manager [req-a9e2db98-3cc8-4ab8-9fb1-2d5897f75a73 req-8852b2b7-5e60-4829-a03c-5530dc7194c6 service nova] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Refreshing instance network info cache due to event network-changed-39337e32-ee39-47e5-ad79-389ade158ef2. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 693.054087] env[62499]: DEBUG oslo_concurrency.lockutils [req-a9e2db98-3cc8-4ab8-9fb1-2d5897f75a73 req-8852b2b7-5e60-4829-a03c-5530dc7194c6 service nova] Acquiring lock "refresh_cache-3b5fb42c-8a81-4575-ba8f-424a101d11ae" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 693.054087] env[62499]: DEBUG oslo_concurrency.lockutils [req-a9e2db98-3cc8-4ab8-9fb1-2d5897f75a73 req-8852b2b7-5e60-4829-a03c-5530dc7194c6 service nova] Acquired lock "refresh_cache-3b5fb42c-8a81-4575-ba8f-424a101d11ae" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 693.054087] env[62499]: DEBUG nova.network.neutron [req-a9e2db98-3cc8-4ab8-9fb1-2d5897f75a73 req-8852b2b7-5e60-4829-a03c-5530dc7194c6 service nova] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Refreshing network info cache for port 39337e32-ee39-47e5-ad79-389ade158ef2 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 693.118310] env[62499]: DEBUG oslo_concurrency.lockutils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.946s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 693.119185] env[62499]: ERROR nova.compute.manager [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 94469936-9e70-45b5-bbde-a295c9e733a7, please check neutron logs for more information. [ 693.119185] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Traceback (most recent call last): [ 693.119185] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 693.119185] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] self.driver.spawn(context, instance, image_meta, [ 693.119185] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 693.119185] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] self._vmops.spawn(context, instance, image_meta, injected_files, [ 693.119185] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 693.119185] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] vm_ref = self.build_virtual_machine(instance, [ 693.119185] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 693.119185] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] vif_infos = vmwarevif.get_vif_info(self._session, [ 693.119185] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 693.119582] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] for vif in network_info: [ 693.119582] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 693.119582] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] return self._sync_wrapper(fn, *args, **kwargs) [ 693.119582] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 693.119582] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] self.wait() [ 693.119582] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 693.119582] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] self[:] = self._gt.wait() [ 693.119582] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 693.119582] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] return self._exit_event.wait() [ 693.119582] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 693.119582] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] current.throw(*self._exc) [ 693.119582] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 693.119582] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] result = function(*args, **kwargs) [ 693.119905] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 693.119905] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] return func(*args, **kwargs) [ 693.119905] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 693.119905] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] raise e [ 693.119905] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 693.119905] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] nwinfo = self.network_api.allocate_for_instance( [ 693.119905] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 693.119905] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] created_port_ids = self._update_ports_for_instance( [ 693.119905] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 693.119905] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] with excutils.save_and_reraise_exception(): [ 693.119905] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 693.119905] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] self.force_reraise() [ 693.119905] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 693.120244] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] raise self.value [ 693.120244] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 693.120244] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] updated_port = self._update_port( [ 693.120244] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 693.120244] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] _ensure_no_port_binding_failure(port) [ 693.120244] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 693.120244] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] raise exception.PortBindingFailed(port_id=port['id']) [ 693.120244] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] nova.exception.PortBindingFailed: Binding failed for port 94469936-9e70-45b5-bbde-a295c9e733a7, please check neutron logs for more information. [ 693.120244] env[62499]: ERROR nova.compute.manager [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] [ 693.120244] env[62499]: DEBUG nova.compute.utils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Binding failed for port 94469936-9e70-45b5-bbde-a295c9e733a7, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 693.122014] env[62499]: DEBUG nova.compute.manager [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Build of instance 28832fde-2434-4d5a-ae62-550b8c0eb829 was re-scheduled: Binding failed for port 94469936-9e70-45b5-bbde-a295c9e733a7, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 693.122550] env[62499]: DEBUG nova.compute.manager [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 693.122868] env[62499]: DEBUG oslo_concurrency.lockutils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Acquiring lock "refresh_cache-28832fde-2434-4d5a-ae62-550b8c0eb829" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 693.123093] env[62499]: DEBUG oslo_concurrency.lockutils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Acquired lock "refresh_cache-28832fde-2434-4d5a-ae62-550b8c0eb829" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 693.123340] env[62499]: DEBUG nova.network.neutron [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 693.124938] env[62499]: DEBUG oslo_concurrency.lockutils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 29.378s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 693.126551] env[62499]: INFO nova.compute.claims [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 693.265705] env[62499]: ERROR nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 39337e32-ee39-47e5-ad79-389ade158ef2, please check neutron logs for more information. [ 693.265705] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 693.265705] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 693.265705] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 693.265705] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 693.265705] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 693.265705] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 693.265705] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 693.265705] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 693.265705] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 693.265705] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 693.265705] env[62499]: ERROR nova.compute.manager raise self.value [ 693.265705] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 693.265705] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 693.265705] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 693.265705] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 693.266200] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 693.266200] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 693.266200] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 39337e32-ee39-47e5-ad79-389ade158ef2, please check neutron logs for more information. [ 693.266200] env[62499]: ERROR nova.compute.manager [ 693.266200] env[62499]: Traceback (most recent call last): [ 693.266200] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 693.266200] env[62499]: listener.cb(fileno) [ 693.266200] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 693.266200] env[62499]: result = function(*args, **kwargs) [ 693.266200] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 693.266200] env[62499]: return func(*args, **kwargs) [ 693.266200] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 693.266200] env[62499]: raise e [ 693.266200] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 693.266200] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 693.266200] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 693.266200] env[62499]: created_port_ids = self._update_ports_for_instance( [ 693.266200] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 693.266200] env[62499]: with excutils.save_and_reraise_exception(): [ 693.266200] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 693.266200] env[62499]: self.force_reraise() [ 693.266200] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 693.266200] env[62499]: raise self.value [ 693.266200] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 693.266200] env[62499]: updated_port = self._update_port( [ 693.266200] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 693.266200] env[62499]: _ensure_no_port_binding_failure(port) [ 693.266200] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 693.266200] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 693.267052] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 39337e32-ee39-47e5-ad79-389ade158ef2, please check neutron logs for more information. [ 693.267052] env[62499]: Removing descriptor: 17 [ 693.371165] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 693.572786] env[62499]: DEBUG nova.network.neutron [req-a9e2db98-3cc8-4ab8-9fb1-2d5897f75a73 req-8852b2b7-5e60-4829-a03c-5530dc7194c6 service nova] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 693.643095] env[62499]: DEBUG nova.network.neutron [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 693.678347] env[62499]: DEBUG nova.network.neutron [req-a9e2db98-3cc8-4ab8-9fb1-2d5897f75a73 req-8852b2b7-5e60-4829-a03c-5530dc7194c6 service nova] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 693.734108] env[62499]: DEBUG nova.network.neutron [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 694.182311] env[62499]: DEBUG oslo_concurrency.lockutils [req-a9e2db98-3cc8-4ab8-9fb1-2d5897f75a73 req-8852b2b7-5e60-4829-a03c-5530dc7194c6 service nova] Releasing lock "refresh_cache-3b5fb42c-8a81-4575-ba8f-424a101d11ae" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 694.238147] env[62499]: DEBUG oslo_concurrency.lockutils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Releasing lock "refresh_cache-28832fde-2434-4d5a-ae62-550b8c0eb829" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 694.238147] env[62499]: DEBUG nova.compute.manager [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 694.238147] env[62499]: DEBUG nova.compute.manager [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 694.238147] env[62499]: DEBUG nova.network.neutron [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 694.252554] env[62499]: DEBUG nova.network.neutron [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 694.553131] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f083bb57-3d14-4378-8cc3-df4937c11e7e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.560403] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb2b2274-7dd8-4545-9100-f3e74b243b51 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.593662] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d25e113-7fcf-4685-acf5-1410e2fb117f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.601015] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-687dbc22-2531-46c5-87b6-8405044630be {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.614283] env[62499]: DEBUG nova.compute.provider_tree [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 694.755627] env[62499]: DEBUG nova.network.neutron [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 694.943533] env[62499]: DEBUG nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 694.944032] env[62499]: DEBUG nova.virt.hardware [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum=,container_format=,created_at=,direct_url=,disk_format=,id=,min_disk=0,min_ram=0,name=,owner=,properties=ImageMetaProps,protected=,size=1073741824,status='active',tags=,updated_at=,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 694.944255] env[62499]: DEBUG nova.virt.hardware [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 694.944411] env[62499]: DEBUG nova.virt.hardware [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 694.944589] env[62499]: DEBUG nova.virt.hardware [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 694.944774] env[62499]: DEBUG nova.virt.hardware [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 694.944878] env[62499]: DEBUG nova.virt.hardware [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 694.945091] env[62499]: DEBUG nova.virt.hardware [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 694.945254] env[62499]: DEBUG nova.virt.hardware [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 694.945422] env[62499]: DEBUG nova.virt.hardware [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 694.945586] env[62499]: DEBUG nova.virt.hardware [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 694.945758] env[62499]: DEBUG nova.virt.hardware [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 694.946629] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-785ec5c5-2ffc-47fe-bd18-76901ed69a13 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.955525] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0746d13a-f190-4621-9773-83d5bfe8aae0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.968168] env[62499]: ERROR nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 39337e32-ee39-47e5-ad79-389ade158ef2, please check neutron logs for more information. [ 694.968168] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Traceback (most recent call last): [ 694.968168] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 694.968168] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] yield resources [ 694.968168] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 694.968168] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] self.driver.spawn(context, instance, image_meta, [ 694.968168] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 694.968168] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] self._vmops.spawn(context, instance, image_meta, injected_files, [ 694.968168] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 694.968168] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] vm_ref = self.build_virtual_machine(instance, [ 694.968168] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 694.968599] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] vif_infos = vmwarevif.get_vif_info(self._session, [ 694.968599] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 694.968599] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] for vif in network_info: [ 694.968599] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 694.968599] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] return self._sync_wrapper(fn, *args, **kwargs) [ 694.968599] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 694.968599] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] self.wait() [ 694.968599] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 694.968599] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] self[:] = self._gt.wait() [ 694.968599] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 694.968599] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] return self._exit_event.wait() [ 694.968599] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 694.968599] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] current.throw(*self._exc) [ 694.968937] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 694.968937] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] result = function(*args, **kwargs) [ 694.968937] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 694.968937] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] return func(*args, **kwargs) [ 694.968937] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 694.968937] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] raise e [ 694.968937] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 694.968937] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] nwinfo = self.network_api.allocate_for_instance( [ 694.968937] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 694.968937] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] created_port_ids = self._update_ports_for_instance( [ 694.968937] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 694.968937] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] with excutils.save_and_reraise_exception(): [ 694.968937] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 694.969327] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] self.force_reraise() [ 694.969327] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 694.969327] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] raise self.value [ 694.969327] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 694.969327] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] updated_port = self._update_port( [ 694.969327] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 694.969327] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] _ensure_no_port_binding_failure(port) [ 694.969327] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 694.969327] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] raise exception.PortBindingFailed(port_id=port['id']) [ 694.969327] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] nova.exception.PortBindingFailed: Binding failed for port 39337e32-ee39-47e5-ad79-389ade158ef2, please check neutron logs for more information. [ 694.969327] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] [ 694.969327] env[62499]: INFO nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Terminating instance [ 694.970450] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Acquiring lock "refresh_cache-3b5fb42c-8a81-4575-ba8f-424a101d11ae" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 694.970609] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Acquired lock "refresh_cache-3b5fb42c-8a81-4575-ba8f-424a101d11ae" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 694.970772] env[62499]: DEBUG nova.network.neutron [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 695.074027] env[62499]: DEBUG nova.compute.manager [req-b19fbe0c-e724-493e-988b-23ea145516ac req-b12bf842-29a6-4a5a-9cff-e99262da5397 service nova] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Received event network-vif-deleted-39337e32-ee39-47e5-ad79-389ade158ef2 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 695.117651] env[62499]: DEBUG nova.scheduler.client.report [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 695.258643] env[62499]: INFO nova.compute.manager [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] [instance: 28832fde-2434-4d5a-ae62-550b8c0eb829] Took 1.02 seconds to deallocate network for instance. [ 695.489436] env[62499]: DEBUG nova.network.neutron [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 695.593061] env[62499]: DEBUG nova.network.neutron [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 695.622222] env[62499]: DEBUG oslo_concurrency.lockutils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.497s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 695.622730] env[62499]: DEBUG nova.compute.manager [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 695.625230] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 30.408s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 695.629054] env[62499]: INFO nova.compute.claims [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 696.099471] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Releasing lock "refresh_cache-3b5fb42c-8a81-4575-ba8f-424a101d11ae" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 696.100064] env[62499]: DEBUG nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 696.100423] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-18c7c1b9-f7a9-4871-9247-6c9d97c574c7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 696.109562] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76e9b110-81c4-428b-ad92-521ff353b956 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 696.131609] env[62499]: DEBUG nova.compute.utils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 696.135066] env[62499]: WARNING nova.virt.vmwareapi.driver [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Instance does not exists. Proceeding to delete instance properties on datastore: nova.exception.InstanceNotFound: Instance 3b5fb42c-8a81-4575-ba8f-424a101d11ae could not be found. [ 696.135066] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 696.135448] env[62499]: DEBUG nova.compute.manager [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 696.135611] env[62499]: DEBUG nova.network.neutron [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 696.137684] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-08557fef-182c-4135-81e0-2e9fdf688eeb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 696.145776] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e57d2c68-6716-4a82-ac12-df804c92be14 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 696.167562] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3b5fb42c-8a81-4575-ba8f-424a101d11ae could not be found. [ 696.167654] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 696.167787] env[62499]: INFO nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Took 0.07 seconds to destroy the instance on the hypervisor. [ 696.168040] env[62499]: DEBUG oslo.service.loopingcall [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 696.168260] env[62499]: DEBUG nova.compute.manager [-] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 696.168354] env[62499]: DEBUG nova.network.neutron [-] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 696.182645] env[62499]: DEBUG nova.network.neutron [-] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 696.199211] env[62499]: DEBUG nova.policy [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '8d900d87cd0b4d3f83ca573311a26304', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '2560c0a335224866815786b4185bb761', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 696.297022] env[62499]: INFO nova.scheduler.client.report [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Deleted allocations for instance 28832fde-2434-4d5a-ae62-550b8c0eb829 [ 696.505241] env[62499]: DEBUG nova.network.neutron [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Successfully created port: ddd7b2bb-0a7c-4f1f-ad88-162c46df78b1 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 696.642460] env[62499]: DEBUG nova.compute.manager [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 696.688106] env[62499]: DEBUG nova.network.neutron [-] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 696.804211] env[62499]: DEBUG oslo_concurrency.lockutils [None req-90b2d7e4-f5f9-4c6a-b2e4-0fe27ec60361 tempest-ServerRescueTestJSON-1614811012 tempest-ServerRescueTestJSON-1614811012-project-member] Lock "28832fde-2434-4d5a-ae62-550b8c0eb829" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 105.664s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 697.061370] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ef26dae-b867-4169-8527-cb67365828b6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 697.068991] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e864c32-c0c3-4fec-b91f-3f023c116fba {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 697.104170] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc57d63c-6694-4657-87cd-6897ee16e943 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 697.112602] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d99ec90e-cf1a-4ad2-bf92-93188e6d0c36 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 697.127076] env[62499]: DEBUG nova.compute.provider_tree [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 697.191426] env[62499]: INFO nova.compute.manager [-] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Took 1.02 seconds to deallocate network for instance. [ 697.306966] env[62499]: DEBUG nova.compute.manager [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 697.494085] env[62499]: DEBUG nova.compute.manager [req-2a538382-33b2-4530-8b79-b9a81212a4b1 req-fa5d5466-dcea-4532-8027-49ae07f00f44 service nova] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Received event network-changed-ddd7b2bb-0a7c-4f1f-ad88-162c46df78b1 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 697.494303] env[62499]: DEBUG nova.compute.manager [req-2a538382-33b2-4530-8b79-b9a81212a4b1 req-fa5d5466-dcea-4532-8027-49ae07f00f44 service nova] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Refreshing instance network info cache due to event network-changed-ddd7b2bb-0a7c-4f1f-ad88-162c46df78b1. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 697.494520] env[62499]: DEBUG oslo_concurrency.lockutils [req-2a538382-33b2-4530-8b79-b9a81212a4b1 req-fa5d5466-dcea-4532-8027-49ae07f00f44 service nova] Acquiring lock "refresh_cache-0899249c-880c-48de-b2c3-2c407115463e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 697.494748] env[62499]: DEBUG oslo_concurrency.lockutils [req-2a538382-33b2-4530-8b79-b9a81212a4b1 req-fa5d5466-dcea-4532-8027-49ae07f00f44 service nova] Acquired lock "refresh_cache-0899249c-880c-48de-b2c3-2c407115463e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 697.494859] env[62499]: DEBUG nova.network.neutron [req-2a538382-33b2-4530-8b79-b9a81212a4b1 req-fa5d5466-dcea-4532-8027-49ae07f00f44 service nova] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Refreshing network info cache for port ddd7b2bb-0a7c-4f1f-ad88-162c46df78b1 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 697.632070] env[62499]: DEBUG nova.scheduler.client.report [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 697.656968] env[62499]: DEBUG nova.compute.manager [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 697.663189] env[62499]: ERROR nova.compute.manager [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ddd7b2bb-0a7c-4f1f-ad88-162c46df78b1, please check neutron logs for more information. [ 697.663189] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 697.663189] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 697.663189] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 697.663189] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 697.663189] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 697.663189] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 697.663189] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 697.663189] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 697.663189] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 697.663189] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 697.663189] env[62499]: ERROR nova.compute.manager raise self.value [ 697.663189] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 697.663189] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 697.663189] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 697.663189] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 697.663939] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 697.663939] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 697.663939] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ddd7b2bb-0a7c-4f1f-ad88-162c46df78b1, please check neutron logs for more information. [ 697.663939] env[62499]: ERROR nova.compute.manager [ 697.663939] env[62499]: Traceback (most recent call last): [ 697.663939] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 697.663939] env[62499]: listener.cb(fileno) [ 697.663939] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 697.663939] env[62499]: result = function(*args, **kwargs) [ 697.663939] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 697.663939] env[62499]: return func(*args, **kwargs) [ 697.663939] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 697.663939] env[62499]: raise e [ 697.663939] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 697.663939] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 697.663939] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 697.663939] env[62499]: created_port_ids = self._update_ports_for_instance( [ 697.663939] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 697.663939] env[62499]: with excutils.save_and_reraise_exception(): [ 697.663939] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 697.663939] env[62499]: self.force_reraise() [ 697.663939] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 697.663939] env[62499]: raise self.value [ 697.663939] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 697.663939] env[62499]: updated_port = self._update_port( [ 697.663939] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 697.663939] env[62499]: _ensure_no_port_binding_failure(port) [ 697.663939] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 697.663939] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 697.665304] env[62499]: nova.exception.PortBindingFailed: Binding failed for port ddd7b2bb-0a7c-4f1f-ad88-162c46df78b1, please check neutron logs for more information. [ 697.665304] env[62499]: Removing descriptor: 16 [ 697.684418] env[62499]: DEBUG nova.virt.hardware [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 697.684894] env[62499]: DEBUG nova.virt.hardware [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 697.685649] env[62499]: DEBUG nova.virt.hardware [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 697.685649] env[62499]: DEBUG nova.virt.hardware [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 697.685649] env[62499]: DEBUG nova.virt.hardware [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 697.685814] env[62499]: DEBUG nova.virt.hardware [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 697.687304] env[62499]: DEBUG nova.virt.hardware [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 697.687304] env[62499]: DEBUG nova.virt.hardware [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 697.687304] env[62499]: DEBUG nova.virt.hardware [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 697.687304] env[62499]: DEBUG nova.virt.hardware [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 697.687304] env[62499]: DEBUG nova.virt.hardware [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 697.688902] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d81254a-d98e-4803-a5d9-184adceb7f6a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 697.696668] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bb7335e2-83d6-4ee7-96b5-ece34a4ef932 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 697.710416] env[62499]: ERROR nova.compute.manager [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ddd7b2bb-0a7c-4f1f-ad88-162c46df78b1, please check neutron logs for more information. [ 697.710416] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] Traceback (most recent call last): [ 697.710416] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 697.710416] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] yield resources [ 697.710416] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 697.710416] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] self.driver.spawn(context, instance, image_meta, [ 697.710416] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 697.710416] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 697.710416] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 697.710416] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] vm_ref = self.build_virtual_machine(instance, [ 697.710416] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 697.710839] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] vif_infos = vmwarevif.get_vif_info(self._session, [ 697.710839] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 697.710839] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] for vif in network_info: [ 697.710839] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 697.710839] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] return self._sync_wrapper(fn, *args, **kwargs) [ 697.710839] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 697.710839] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] self.wait() [ 697.710839] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 697.710839] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] self[:] = self._gt.wait() [ 697.710839] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 697.710839] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] return self._exit_event.wait() [ 697.710839] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 697.710839] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] current.throw(*self._exc) [ 697.711192] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 697.711192] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] result = function(*args, **kwargs) [ 697.711192] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 697.711192] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] return func(*args, **kwargs) [ 697.711192] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 697.711192] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] raise e [ 697.711192] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 697.711192] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] nwinfo = self.network_api.allocate_for_instance( [ 697.711192] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 697.711192] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] created_port_ids = self._update_ports_for_instance( [ 697.711192] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 697.711192] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] with excutils.save_and_reraise_exception(): [ 697.711192] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 697.711560] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] self.force_reraise() [ 697.711560] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 697.711560] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] raise self.value [ 697.711560] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 697.711560] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] updated_port = self._update_port( [ 697.711560] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 697.711560] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] _ensure_no_port_binding_failure(port) [ 697.711560] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 697.711560] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] raise exception.PortBindingFailed(port_id=port['id']) [ 697.711560] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] nova.exception.PortBindingFailed: Binding failed for port ddd7b2bb-0a7c-4f1f-ad88-162c46df78b1, please check neutron logs for more information. [ 697.711560] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] [ 697.711560] env[62499]: INFO nova.compute.manager [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Terminating instance [ 697.712728] env[62499]: DEBUG oslo_concurrency.lockutils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Acquiring lock "refresh_cache-0899249c-880c-48de-b2c3-2c407115463e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 697.767901] env[62499]: INFO nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Took 0.58 seconds to detach 1 volumes for instance. [ 697.771599] env[62499]: DEBUG nova.compute.claims [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 697.771801] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 697.831935] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 698.026214] env[62499]: DEBUG nova.network.neutron [req-2a538382-33b2-4530-8b79-b9a81212a4b1 req-fa5d5466-dcea-4532-8027-49ae07f00f44 service nova] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 698.130642] env[62499]: DEBUG nova.network.neutron [req-2a538382-33b2-4530-8b79-b9a81212a4b1 req-fa5d5466-dcea-4532-8027-49ae07f00f44 service nova] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 698.141255] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.515s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 698.141255] env[62499]: DEBUG nova.compute.manager [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 698.146018] env[62499]: DEBUG oslo_concurrency.lockutils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 29.465s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 698.146018] env[62499]: INFO nova.compute.claims [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 698.640299] env[62499]: DEBUG oslo_concurrency.lockutils [req-2a538382-33b2-4530-8b79-b9a81212a4b1 req-fa5d5466-dcea-4532-8027-49ae07f00f44 service nova] Releasing lock "refresh_cache-0899249c-880c-48de-b2c3-2c407115463e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 698.640756] env[62499]: DEBUG oslo_concurrency.lockutils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Acquired lock "refresh_cache-0899249c-880c-48de-b2c3-2c407115463e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 698.640990] env[62499]: DEBUG nova.network.neutron [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 698.648065] env[62499]: DEBUG nova.compute.utils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 698.652333] env[62499]: DEBUG nova.compute.manager [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 698.652510] env[62499]: DEBUG nova.network.neutron [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 698.705451] env[62499]: DEBUG nova.policy [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '8a58d14b9cc84049a40b16654df2cbdd', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '67dfc1220d0e4a118bf966b471761b74', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 699.004054] env[62499]: DEBUG nova.network.neutron [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Successfully created port: da8078a3-f87c-46cd-ad54-61d196be63f0 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 699.154406] env[62499]: DEBUG nova.compute.manager [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 699.166086] env[62499]: DEBUG nova.network.neutron [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 699.377039] env[62499]: DEBUG nova.network.neutron [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 699.540099] env[62499]: DEBUG nova.compute.manager [req-afec7013-2b89-4c05-89e4-b5b1ad046dbf req-26763264-1f4d-4bc8-80a0-86f22e061e5a service nova] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Received event network-vif-deleted-ddd7b2bb-0a7c-4f1f-ad88-162c46df78b1 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 699.608026] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a52bb08e-e5bf-4b54-8ed4-13fa3b5b0957 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.616919] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67e28f51-82bf-4171-ab61-bb30aa92fbe8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.652973] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c0a34799-1b62-42af-ae4b-163b7d79c8d4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.662222] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59930841-b6f9-4abf-b585-3cf1cc49ea52 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.679781] env[62499]: DEBUG nova.compute.provider_tree [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 699.881424] env[62499]: DEBUG oslo_concurrency.lockutils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Releasing lock "refresh_cache-0899249c-880c-48de-b2c3-2c407115463e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 699.881846] env[62499]: DEBUG nova.compute.manager [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 699.882037] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 699.882333] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d1ed090e-b15c-42fd-b3e1-9b26dd563e59 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.893212] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f3b9b68-9133-446e-b261-2ca19e318314 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.924285] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 0899249c-880c-48de-b2c3-2c407115463e could not be found. [ 699.924546] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 699.924733] env[62499]: INFO nova.compute.manager [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Took 0.04 seconds to destroy the instance on the hypervisor. [ 699.924982] env[62499]: DEBUG oslo.service.loopingcall [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 699.925227] env[62499]: DEBUG nova.compute.manager [-] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 699.925323] env[62499]: DEBUG nova.network.neutron [-] [instance: 0899249c-880c-48de-b2c3-2c407115463e] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 699.943497] env[62499]: DEBUG nova.network.neutron [-] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 700.090016] env[62499]: ERROR nova.compute.manager [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port da8078a3-f87c-46cd-ad54-61d196be63f0, please check neutron logs for more information. [ 700.090016] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 700.090016] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 700.090016] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 700.090016] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 700.090016] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 700.090016] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 700.090016] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 700.090016] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 700.090016] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 700.090016] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 700.090016] env[62499]: ERROR nova.compute.manager raise self.value [ 700.090016] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 700.090016] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 700.090016] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 700.090016] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 700.090490] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 700.090490] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 700.090490] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port da8078a3-f87c-46cd-ad54-61d196be63f0, please check neutron logs for more information. [ 700.090490] env[62499]: ERROR nova.compute.manager [ 700.090490] env[62499]: Traceback (most recent call last): [ 700.090490] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 700.090490] env[62499]: listener.cb(fileno) [ 700.090490] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 700.090490] env[62499]: result = function(*args, **kwargs) [ 700.090490] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 700.090490] env[62499]: return func(*args, **kwargs) [ 700.090490] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 700.090490] env[62499]: raise e [ 700.090490] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 700.090490] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 700.090490] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 700.090490] env[62499]: created_port_ids = self._update_ports_for_instance( [ 700.090490] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 700.090490] env[62499]: with excutils.save_and_reraise_exception(): [ 700.090490] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 700.090490] env[62499]: self.force_reraise() [ 700.090490] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 700.090490] env[62499]: raise self.value [ 700.090490] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 700.090490] env[62499]: updated_port = self._update_port( [ 700.090490] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 700.090490] env[62499]: _ensure_no_port_binding_failure(port) [ 700.090490] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 700.090490] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 700.091279] env[62499]: nova.exception.PortBindingFailed: Binding failed for port da8078a3-f87c-46cd-ad54-61d196be63f0, please check neutron logs for more information. [ 700.091279] env[62499]: Removing descriptor: 16 [ 700.172855] env[62499]: DEBUG nova.compute.manager [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 700.182809] env[62499]: DEBUG nova.scheduler.client.report [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 700.192743] env[62499]: DEBUG nova.virt.hardware [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 700.192977] env[62499]: DEBUG nova.virt.hardware [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 700.193149] env[62499]: DEBUG nova.virt.hardware [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 700.193334] env[62499]: DEBUG nova.virt.hardware [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 700.193481] env[62499]: DEBUG nova.virt.hardware [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 700.193630] env[62499]: DEBUG nova.virt.hardware [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 700.193831] env[62499]: DEBUG nova.virt.hardware [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 700.193985] env[62499]: DEBUG nova.virt.hardware [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 700.194166] env[62499]: DEBUG nova.virt.hardware [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 700.194324] env[62499]: DEBUG nova.virt.hardware [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 700.194496] env[62499]: DEBUG nova.virt.hardware [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 700.196450] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c44f8f9a-8131-4bba-adc2-9c4ed314ece8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 700.204539] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40a880ba-3bd8-4eae-a200-8305ada6dc49 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 700.228028] env[62499]: ERROR nova.compute.manager [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port da8078a3-f87c-46cd-ad54-61d196be63f0, please check neutron logs for more information. [ 700.228028] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Traceback (most recent call last): [ 700.228028] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 700.228028] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] yield resources [ 700.228028] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 700.228028] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] self.driver.spawn(context, instance, image_meta, [ 700.228028] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 700.228028] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 700.228028] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 700.228028] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] vm_ref = self.build_virtual_machine(instance, [ 700.228028] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 700.228451] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] vif_infos = vmwarevif.get_vif_info(self._session, [ 700.228451] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 700.228451] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] for vif in network_info: [ 700.228451] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 700.228451] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] return self._sync_wrapper(fn, *args, **kwargs) [ 700.228451] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 700.228451] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] self.wait() [ 700.228451] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 700.228451] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] self[:] = self._gt.wait() [ 700.228451] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 700.228451] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] return self._exit_event.wait() [ 700.228451] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 700.228451] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] current.throw(*self._exc) [ 700.228845] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 700.228845] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] result = function(*args, **kwargs) [ 700.228845] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 700.228845] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] return func(*args, **kwargs) [ 700.228845] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 700.228845] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] raise e [ 700.228845] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 700.228845] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] nwinfo = self.network_api.allocate_for_instance( [ 700.228845] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 700.228845] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] created_port_ids = self._update_ports_for_instance( [ 700.228845] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 700.228845] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] with excutils.save_and_reraise_exception(): [ 700.228845] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 700.229280] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] self.force_reraise() [ 700.229280] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 700.229280] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] raise self.value [ 700.229280] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 700.229280] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] updated_port = self._update_port( [ 700.229280] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 700.229280] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] _ensure_no_port_binding_failure(port) [ 700.229280] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 700.229280] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] raise exception.PortBindingFailed(port_id=port['id']) [ 700.229280] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] nova.exception.PortBindingFailed: Binding failed for port da8078a3-f87c-46cd-ad54-61d196be63f0, please check neutron logs for more information. [ 700.229280] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] [ 700.229280] env[62499]: INFO nova.compute.manager [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Terminating instance [ 700.231422] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Acquiring lock "refresh_cache-a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 700.231658] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Acquired lock "refresh_cache-a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 700.231828] env[62499]: DEBUG nova.network.neutron [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 700.446796] env[62499]: DEBUG nova.network.neutron [-] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 700.689597] env[62499]: DEBUG oslo_concurrency.lockutils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.547s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 700.689972] env[62499]: DEBUG nova.compute.manager [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 700.693705] env[62499]: DEBUG oslo_concurrency.lockutils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 30.134s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 700.757297] env[62499]: DEBUG nova.network.neutron [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 700.884102] env[62499]: DEBUG nova.network.neutron [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 700.951276] env[62499]: INFO nova.compute.manager [-] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Took 1.03 seconds to deallocate network for instance. [ 700.953788] env[62499]: DEBUG nova.compute.claims [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 700.953964] env[62499]: DEBUG oslo_concurrency.lockutils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 700.994568] env[62499]: DEBUG oslo_concurrency.lockutils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Acquiring lock "261cbefa-fb2d-48da-a4fe-80b744a931f6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 700.994848] env[62499]: DEBUG oslo_concurrency.lockutils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Lock "261cbefa-fb2d-48da-a4fe-80b744a931f6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 701.196507] env[62499]: DEBUG oslo_concurrency.lockutils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Acquiring lock "49f9108c-7256-4a2c-9ffd-a6d041a180e4" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 701.196786] env[62499]: DEBUG oslo_concurrency.lockutils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Lock "49f9108c-7256-4a2c-9ffd-a6d041a180e4" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 701.204624] env[62499]: DEBUG nova.compute.utils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 701.206499] env[62499]: DEBUG nova.compute.manager [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 701.206499] env[62499]: DEBUG nova.network.neutron [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 701.257770] env[62499]: DEBUG nova.policy [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a4763a330d70462182e5bdccc2a08c25', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9512415a4f124e8c9120f6c115f78949', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 701.386814] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Releasing lock "refresh_cache-a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 701.387262] env[62499]: DEBUG nova.compute.manager [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 701.387816] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 701.387816] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-201453b4-6548-4a0e-8410-ae0958bebbff {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.397219] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-496e2063-5acf-4ac2-8a29-014b5cb2aedd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.426930] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b could not be found. [ 701.427205] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 701.427405] env[62499]: INFO nova.compute.manager [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Took 0.04 seconds to destroy the instance on the hypervisor. [ 701.427651] env[62499]: DEBUG oslo.service.loopingcall [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 701.430799] env[62499]: DEBUG nova.compute.manager [-] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 701.430891] env[62499]: DEBUG nova.network.neutron [-] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 701.471927] env[62499]: DEBUG nova.network.neutron [-] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 701.579125] env[62499]: DEBUG nova.compute.manager [req-6977c6c5-e682-4149-bac4-df2ab442c232 req-2f29a048-9d6d-4f39-8b6e-ba0980fe75bd service nova] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Received event network-changed-da8078a3-f87c-46cd-ad54-61d196be63f0 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 701.579313] env[62499]: DEBUG nova.compute.manager [req-6977c6c5-e682-4149-bac4-df2ab442c232 req-2f29a048-9d6d-4f39-8b6e-ba0980fe75bd service nova] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Refreshing instance network info cache due to event network-changed-da8078a3-f87c-46cd-ad54-61d196be63f0. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 701.579527] env[62499]: DEBUG oslo_concurrency.lockutils [req-6977c6c5-e682-4149-bac4-df2ab442c232 req-2f29a048-9d6d-4f39-8b6e-ba0980fe75bd service nova] Acquiring lock "refresh_cache-a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 701.579672] env[62499]: DEBUG oslo_concurrency.lockutils [req-6977c6c5-e682-4149-bac4-df2ab442c232 req-2f29a048-9d6d-4f39-8b6e-ba0980fe75bd service nova] Acquired lock "refresh_cache-a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 701.579905] env[62499]: DEBUG nova.network.neutron [req-6977c6c5-e682-4149-bac4-df2ab442c232 req-2f29a048-9d6d-4f39-8b6e-ba0980fe75bd service nova] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Refreshing network info cache for port da8078a3-f87c-46cd-ad54-61d196be63f0 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 701.613112] env[62499]: DEBUG nova.network.neutron [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Successfully created port: 993e00ef-a852-4091-bc3a-7acc78478536 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 701.708975] env[62499]: DEBUG nova.compute.manager [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 701.712590] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59982ec2-3d12-44d1-9c09-990c0ff9192e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.724159] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f54ed5a4-072e-4cf1-85e8-636cf519d062 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.756380] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c3cd0e6-4304-4fe8-9bba-4ceab80e9a03 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.765748] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cea3e281-5e6a-4d14-b077-d96caa420e0f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.780972] env[62499]: DEBUG nova.compute.provider_tree [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 701.975465] env[62499]: DEBUG nova.network.neutron [-] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 702.105755] env[62499]: DEBUG nova.network.neutron [req-6977c6c5-e682-4149-bac4-df2ab442c232 req-2f29a048-9d6d-4f39-8b6e-ba0980fe75bd service nova] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 702.230239] env[62499]: DEBUG nova.network.neutron [req-6977c6c5-e682-4149-bac4-df2ab442c232 req-2f29a048-9d6d-4f39-8b6e-ba0980fe75bd service nova] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 702.283477] env[62499]: DEBUG nova.scheduler.client.report [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 702.480012] env[62499]: INFO nova.compute.manager [-] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Took 1.05 seconds to deallocate network for instance. [ 702.481094] env[62499]: DEBUG nova.compute.claims [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 702.481094] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 702.731894] env[62499]: DEBUG nova.compute.manager [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 702.737234] env[62499]: DEBUG oslo_concurrency.lockutils [req-6977c6c5-e682-4149-bac4-df2ab442c232 req-2f29a048-9d6d-4f39-8b6e-ba0980fe75bd service nova] Releasing lock "refresh_cache-a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 702.737234] env[62499]: DEBUG nova.compute.manager [req-6977c6c5-e682-4149-bac4-df2ab442c232 req-2f29a048-9d6d-4f39-8b6e-ba0980fe75bd service nova] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Received event network-vif-deleted-da8078a3-f87c-46cd-ad54-61d196be63f0 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 702.769804] env[62499]: ERROR nova.compute.manager [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 993e00ef-a852-4091-bc3a-7acc78478536, please check neutron logs for more information. [ 702.769804] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 702.769804] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 702.769804] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 702.769804] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 702.769804] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 702.769804] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 702.769804] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 702.769804] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 702.769804] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 702.769804] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 702.769804] env[62499]: ERROR nova.compute.manager raise self.value [ 702.769804] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 702.769804] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 702.769804] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 702.769804] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 702.770360] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 702.770360] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 702.770360] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 993e00ef-a852-4091-bc3a-7acc78478536, please check neutron logs for more information. [ 702.770360] env[62499]: ERROR nova.compute.manager [ 702.770360] env[62499]: Traceback (most recent call last): [ 702.770360] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 702.770360] env[62499]: listener.cb(fileno) [ 702.770360] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 702.770360] env[62499]: result = function(*args, **kwargs) [ 702.770360] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 702.770360] env[62499]: return func(*args, **kwargs) [ 702.770360] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 702.770360] env[62499]: raise e [ 702.770360] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 702.770360] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 702.770360] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 702.770360] env[62499]: created_port_ids = self._update_ports_for_instance( [ 702.770360] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 702.770360] env[62499]: with excutils.save_and_reraise_exception(): [ 702.770360] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 702.770360] env[62499]: self.force_reraise() [ 702.770360] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 702.770360] env[62499]: raise self.value [ 702.770360] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 702.770360] env[62499]: updated_port = self._update_port( [ 702.770360] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 702.770360] env[62499]: _ensure_no_port_binding_failure(port) [ 702.770360] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 702.770360] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 702.771596] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 993e00ef-a852-4091-bc3a-7acc78478536, please check neutron logs for more information. [ 702.771596] env[62499]: Removing descriptor: 16 [ 702.777053] env[62499]: DEBUG nova.virt.hardware [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 702.777053] env[62499]: DEBUG nova.virt.hardware [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 702.777053] env[62499]: DEBUG nova.virt.hardware [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 702.777466] env[62499]: DEBUG nova.virt.hardware [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 702.777466] env[62499]: DEBUG nova.virt.hardware [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 702.777466] env[62499]: DEBUG nova.virt.hardware [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 702.777466] env[62499]: DEBUG nova.virt.hardware [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 702.777466] env[62499]: DEBUG nova.virt.hardware [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 702.777714] env[62499]: DEBUG nova.virt.hardware [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 702.777714] env[62499]: DEBUG nova.virt.hardware [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 702.777714] env[62499]: DEBUG nova.virt.hardware [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 702.778331] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-826fe183-f0b7-44b8-90c1-0ce30ef02dbc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 702.788413] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Acquiring lock "d650a1a5-3706-4682-a813-f85ea23098e2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 702.788733] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Lock "d650a1a5-3706-4682-a813-f85ea23098e2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 702.792981] env[62499]: DEBUG oslo_concurrency.lockutils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.100s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 702.793603] env[62499]: ERROR nova.compute.manager [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1dd261e2-15db-4d61-8991-84f77bee309f, please check neutron logs for more information. [ 702.793603] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Traceback (most recent call last): [ 702.793603] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 702.793603] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] self.driver.spawn(context, instance, image_meta, [ 702.793603] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 702.793603] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] self._vmops.spawn(context, instance, image_meta, injected_files, [ 702.793603] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 702.793603] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] vm_ref = self.build_virtual_machine(instance, [ 702.793603] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 702.793603] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] vif_infos = vmwarevif.get_vif_info(self._session, [ 702.793603] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 702.793978] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] for vif in network_info: [ 702.793978] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 702.793978] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] return self._sync_wrapper(fn, *args, **kwargs) [ 702.793978] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 702.793978] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] self.wait() [ 702.793978] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 702.793978] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] self[:] = self._gt.wait() [ 702.793978] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 702.793978] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] return self._exit_event.wait() [ 702.793978] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 702.793978] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] result = hub.switch() [ 702.793978] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 702.793978] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] return self.greenlet.switch() [ 702.794401] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 702.794401] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] result = function(*args, **kwargs) [ 702.794401] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 702.794401] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] return func(*args, **kwargs) [ 702.794401] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 702.794401] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] raise e [ 702.794401] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 702.794401] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] nwinfo = self.network_api.allocate_for_instance( [ 702.794401] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 702.794401] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] created_port_ids = self._update_ports_for_instance( [ 702.794401] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 702.794401] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] with excutils.save_and_reraise_exception(): [ 702.794401] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 702.794808] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] self.force_reraise() [ 702.794808] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 702.794808] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] raise self.value [ 702.794808] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 702.794808] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] updated_port = self._update_port( [ 702.794808] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 702.794808] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] _ensure_no_port_binding_failure(port) [ 702.794808] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 702.794808] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] raise exception.PortBindingFailed(port_id=port['id']) [ 702.794808] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] nova.exception.PortBindingFailed: Binding failed for port 1dd261e2-15db-4d61-8991-84f77bee309f, please check neutron logs for more information. [ 702.794808] env[62499]: ERROR nova.compute.manager [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] [ 702.795562] env[62499]: DEBUG nova.compute.utils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Binding failed for port 1dd261e2-15db-4d61-8991-84f77bee309f, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 702.795851] env[62499]: DEBUG nova.compute.manager [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Build of instance 873731fb-8d97-41e7-830b-b9ae4281a737 was re-scheduled: Binding failed for port 1dd261e2-15db-4d61-8991-84f77bee309f, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 702.796316] env[62499]: DEBUG nova.compute.manager [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 702.796783] env[62499]: DEBUG oslo_concurrency.lockutils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Acquiring lock "refresh_cache-873731fb-8d97-41e7-830b-b9ae4281a737" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 702.796783] env[62499]: DEBUG oslo_concurrency.lockutils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Acquired lock "refresh_cache-873731fb-8d97-41e7-830b-b9ae4281a737" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 702.796783] env[62499]: DEBUG nova.network.neutron [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 702.797754] env[62499]: DEBUG oslo_concurrency.lockutils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 31.019s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 702.799366] env[62499]: INFO nova.compute.claims [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 702.809705] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02c7c210-a0ed-438c-b0fd-51663e6aace4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 702.826032] env[62499]: ERROR nova.compute.manager [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 993e00ef-a852-4091-bc3a-7acc78478536, please check neutron logs for more information. [ 702.826032] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Traceback (most recent call last): [ 702.826032] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 702.826032] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] yield resources [ 702.826032] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 702.826032] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] self.driver.spawn(context, instance, image_meta, [ 702.826032] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 702.826032] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] self._vmops.spawn(context, instance, image_meta, injected_files, [ 702.826032] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 702.826032] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] vm_ref = self.build_virtual_machine(instance, [ 702.826032] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 702.826409] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] vif_infos = vmwarevif.get_vif_info(self._session, [ 702.826409] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 702.826409] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] for vif in network_info: [ 702.826409] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 702.826409] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] return self._sync_wrapper(fn, *args, **kwargs) [ 702.826409] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 702.826409] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] self.wait() [ 702.826409] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 702.826409] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] self[:] = self._gt.wait() [ 702.826409] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 702.826409] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] return self._exit_event.wait() [ 702.826409] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 702.826409] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] current.throw(*self._exc) [ 702.826737] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 702.826737] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] result = function(*args, **kwargs) [ 702.826737] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 702.826737] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] return func(*args, **kwargs) [ 702.826737] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 702.826737] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] raise e [ 702.826737] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 702.826737] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] nwinfo = self.network_api.allocate_for_instance( [ 702.826737] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 702.826737] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] created_port_ids = self._update_ports_for_instance( [ 702.826737] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 702.826737] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] with excutils.save_and_reraise_exception(): [ 702.826737] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 702.827093] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] self.force_reraise() [ 702.827093] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 702.827093] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] raise self.value [ 702.827093] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 702.827093] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] updated_port = self._update_port( [ 702.827093] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 702.827093] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] _ensure_no_port_binding_failure(port) [ 702.827093] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 702.827093] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] raise exception.PortBindingFailed(port_id=port['id']) [ 702.827093] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] nova.exception.PortBindingFailed: Binding failed for port 993e00ef-a852-4091-bc3a-7acc78478536, please check neutron logs for more information. [ 702.827093] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] [ 702.827093] env[62499]: INFO nova.compute.manager [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Terminating instance [ 702.828754] env[62499]: DEBUG oslo_concurrency.lockutils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquiring lock "refresh_cache-3d9ac347-ffcc-4840-8090-811dbd2865ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 702.828754] env[62499]: DEBUG oslo_concurrency.lockutils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquired lock "refresh_cache-3d9ac347-ffcc-4840-8090-811dbd2865ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 702.829038] env[62499]: DEBUG nova.network.neutron [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 702.832013] env[62499]: DEBUG nova.network.neutron [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 702.923069] env[62499]: DEBUG nova.network.neutron [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 703.351015] env[62499]: DEBUG nova.network.neutron [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 703.425111] env[62499]: DEBUG oslo_concurrency.lockutils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Releasing lock "refresh_cache-873731fb-8d97-41e7-830b-b9ae4281a737" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 703.425534] env[62499]: DEBUG nova.compute.manager [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 703.425732] env[62499]: DEBUG nova.compute.manager [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 703.425901] env[62499]: DEBUG nova.network.neutron [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 703.440135] env[62499]: DEBUG nova.network.neutron [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 703.442804] env[62499]: DEBUG nova.network.neutron [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 703.626473] env[62499]: DEBUG nova.compute.manager [req-947027ae-5f07-4885-9fe4-38592a4d50c9 req-7a4c2198-ab4c-4ed9-859c-cb7ffd67ef7e service nova] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Received event network-changed-993e00ef-a852-4091-bc3a-7acc78478536 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 703.626687] env[62499]: DEBUG nova.compute.manager [req-947027ae-5f07-4885-9fe4-38592a4d50c9 req-7a4c2198-ab4c-4ed9-859c-cb7ffd67ef7e service nova] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Refreshing instance network info cache due to event network-changed-993e00ef-a852-4091-bc3a-7acc78478536. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 703.626878] env[62499]: DEBUG oslo_concurrency.lockutils [req-947027ae-5f07-4885-9fe4-38592a4d50c9 req-7a4c2198-ab4c-4ed9-859c-cb7ffd67ef7e service nova] Acquiring lock "refresh_cache-3d9ac347-ffcc-4840-8090-811dbd2865ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 703.947024] env[62499]: DEBUG nova.network.neutron [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 703.947986] env[62499]: DEBUG oslo_concurrency.lockutils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Releasing lock "refresh_cache-3d9ac347-ffcc-4840-8090-811dbd2865ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 703.948378] env[62499]: DEBUG nova.compute.manager [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 703.948568] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 703.949052] env[62499]: DEBUG oslo_concurrency.lockutils [req-947027ae-5f07-4885-9fe4-38592a4d50c9 req-7a4c2198-ab4c-4ed9-859c-cb7ffd67ef7e service nova] Acquired lock "refresh_cache-3d9ac347-ffcc-4840-8090-811dbd2865ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 703.949263] env[62499]: DEBUG nova.network.neutron [req-947027ae-5f07-4885-9fe4-38592a4d50c9 req-7a4c2198-ab4c-4ed9-859c-cb7ffd67ef7e service nova] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Refreshing network info cache for port 993e00ef-a852-4091-bc3a-7acc78478536 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 703.950099] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-43b3eb9f-0b32-4b27-a259-422efc02013a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 703.961185] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db5b2732-dd1b-41ba-b5d1-7bf8a73f200c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 703.985853] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3d9ac347-ffcc-4840-8090-811dbd2865ef could not be found. [ 703.986098] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 703.986279] env[62499]: INFO nova.compute.manager [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Took 0.04 seconds to destroy the instance on the hypervisor. [ 703.986517] env[62499]: DEBUG oslo.service.loopingcall [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 703.988702] env[62499]: DEBUG nova.compute.manager [-] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 703.988819] env[62499]: DEBUG nova.network.neutron [-] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 704.011226] env[62499]: DEBUG nova.network.neutron [-] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 704.191825] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-664b2221-e2ca-4149-aef0-f724bb5260fe {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.199466] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8493b44e-722e-4e5e-b0d8-b839485ed3a3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.228424] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-53978667-10c9-42db-92c6-cab3ef3b25f2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.235840] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-87cb71b8-8f31-43d7-aee4-6e2c7a05894d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.248821] env[62499]: DEBUG nova.compute.provider_tree [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 704.451032] env[62499]: INFO nova.compute.manager [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] [instance: 873731fb-8d97-41e7-830b-b9ae4281a737] Took 1.02 seconds to deallocate network for instance. [ 704.474899] env[62499]: DEBUG nova.network.neutron [req-947027ae-5f07-4885-9fe4-38592a4d50c9 req-7a4c2198-ab4c-4ed9-859c-cb7ffd67ef7e service nova] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 704.514027] env[62499]: DEBUG nova.network.neutron [-] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 704.586526] env[62499]: DEBUG nova.network.neutron [req-947027ae-5f07-4885-9fe4-38592a4d50c9 req-7a4c2198-ab4c-4ed9-859c-cb7ffd67ef7e service nova] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 704.752592] env[62499]: DEBUG nova.scheduler.client.report [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 705.017062] env[62499]: INFO nova.compute.manager [-] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Took 1.03 seconds to deallocate network for instance. [ 705.019242] env[62499]: DEBUG nova.compute.claims [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 705.019465] env[62499]: DEBUG oslo_concurrency.lockutils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 705.089368] env[62499]: DEBUG oslo_concurrency.lockutils [req-947027ae-5f07-4885-9fe4-38592a4d50c9 req-7a4c2198-ab4c-4ed9-859c-cb7ffd67ef7e service nova] Releasing lock "refresh_cache-3d9ac347-ffcc-4840-8090-811dbd2865ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 705.090028] env[62499]: DEBUG nova.compute.manager [req-947027ae-5f07-4885-9fe4-38592a4d50c9 req-7a4c2198-ab4c-4ed9-859c-cb7ffd67ef7e service nova] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Received event network-vif-deleted-993e00ef-a852-4091-bc3a-7acc78478536 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 705.258071] env[62499]: DEBUG oslo_concurrency.lockutils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.460s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 705.258239] env[62499]: DEBUG nova.compute.manager [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 705.260933] env[62499]: DEBUG oslo_concurrency.lockutils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.272s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 705.262359] env[62499]: INFO nova.compute.claims [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 705.485015] env[62499]: INFO nova.scheduler.client.report [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Deleted allocations for instance 873731fb-8d97-41e7-830b-b9ae4281a737 [ 705.766818] env[62499]: DEBUG nova.compute.utils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 705.771906] env[62499]: DEBUG nova.compute.manager [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 705.772122] env[62499]: DEBUG nova.network.neutron [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 705.824280] env[62499]: DEBUG nova.policy [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7f85a963ad4543a2a0cc460b355a9d26', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '96ee79ce9ee544ebb82b5562d682e849', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 705.992139] env[62499]: DEBUG oslo_concurrency.lockutils [None req-25c0b96c-3e18-4388-8cdb-650d7a088322 tempest-ServerActionsTestOtherA-1189380881 tempest-ServerActionsTestOtherA-1189380881-project-member] Lock "873731fb-8d97-41e7-830b-b9ae4281a737" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 110.547s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 706.135171] env[62499]: DEBUG nova.network.neutron [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Successfully created port: 2d6c29d5-c728-42bd-86a1-1f0309315094 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 706.274016] env[62499]: DEBUG nova.compute.manager [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 706.496114] env[62499]: DEBUG nova.compute.manager [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 706.729313] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee65bc96-9d58-4373-85d9-4fbb4aa49e0e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.737480] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f6de3e65-c90a-4379-bcca-83f9cf2b41dd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.770180] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67ba51cf-eb8b-49dc-b613-e02431cf16f0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.778398] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb51869e-92b3-4093-98b7-6250b9bdd7b9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.797303] env[62499]: DEBUG nova.compute.provider_tree [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 707.018813] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 707.146865] env[62499]: DEBUG nova.compute.manager [req-9c8839f9-d503-4250-a6c0-c6ecfc18e686 req-c4c25130-ad4e-4f9e-9df5-27481402b062 service nova] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Received event network-changed-2d6c29d5-c728-42bd-86a1-1f0309315094 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 707.146865] env[62499]: DEBUG nova.compute.manager [req-9c8839f9-d503-4250-a6c0-c6ecfc18e686 req-c4c25130-ad4e-4f9e-9df5-27481402b062 service nova] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Refreshing instance network info cache due to event network-changed-2d6c29d5-c728-42bd-86a1-1f0309315094. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 707.146865] env[62499]: DEBUG oslo_concurrency.lockutils [req-9c8839f9-d503-4250-a6c0-c6ecfc18e686 req-c4c25130-ad4e-4f9e-9df5-27481402b062 service nova] Acquiring lock "refresh_cache-3649431e-f856-4a28-9af7-8f101c3fc2d6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 707.146865] env[62499]: DEBUG oslo_concurrency.lockutils [req-9c8839f9-d503-4250-a6c0-c6ecfc18e686 req-c4c25130-ad4e-4f9e-9df5-27481402b062 service nova] Acquired lock "refresh_cache-3649431e-f856-4a28-9af7-8f101c3fc2d6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 707.146865] env[62499]: DEBUG nova.network.neutron [req-9c8839f9-d503-4250-a6c0-c6ecfc18e686 req-c4c25130-ad4e-4f9e-9df5-27481402b062 service nova] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Refreshing network info cache for port 2d6c29d5-c728-42bd-86a1-1f0309315094 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 707.290009] env[62499]: DEBUG nova.compute.manager [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 707.300209] env[62499]: DEBUG nova.scheduler.client.report [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 707.307155] env[62499]: ERROR nova.compute.manager [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 2d6c29d5-c728-42bd-86a1-1f0309315094, please check neutron logs for more information. [ 707.307155] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 707.307155] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 707.307155] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 707.307155] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 707.307155] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 707.307155] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 707.307155] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 707.307155] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 707.307155] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 707.307155] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 707.307155] env[62499]: ERROR nova.compute.manager raise self.value [ 707.307155] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 707.307155] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 707.307155] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 707.307155] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 707.307722] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 707.307722] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 707.307722] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 2d6c29d5-c728-42bd-86a1-1f0309315094, please check neutron logs for more information. [ 707.307722] env[62499]: ERROR nova.compute.manager [ 707.307722] env[62499]: Traceback (most recent call last): [ 707.307722] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 707.307722] env[62499]: listener.cb(fileno) [ 707.307722] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 707.307722] env[62499]: result = function(*args, **kwargs) [ 707.307722] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 707.307722] env[62499]: return func(*args, **kwargs) [ 707.307722] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 707.307722] env[62499]: raise e [ 707.307722] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 707.307722] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 707.307722] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 707.307722] env[62499]: created_port_ids = self._update_ports_for_instance( [ 707.307722] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 707.307722] env[62499]: with excutils.save_and_reraise_exception(): [ 707.307722] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 707.307722] env[62499]: self.force_reraise() [ 707.307722] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 707.307722] env[62499]: raise self.value [ 707.307722] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 707.307722] env[62499]: updated_port = self._update_port( [ 707.307722] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 707.307722] env[62499]: _ensure_no_port_binding_failure(port) [ 707.307722] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 707.307722] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 707.308656] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 2d6c29d5-c728-42bd-86a1-1f0309315094, please check neutron logs for more information. [ 707.308656] env[62499]: Removing descriptor: 16 [ 707.324387] env[62499]: DEBUG nova.virt.hardware [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 707.324639] env[62499]: DEBUG nova.virt.hardware [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 707.324797] env[62499]: DEBUG nova.virt.hardware [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 707.324978] env[62499]: DEBUG nova.virt.hardware [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 707.325138] env[62499]: DEBUG nova.virt.hardware [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 707.325285] env[62499]: DEBUG nova.virt.hardware [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 707.325753] env[62499]: DEBUG nova.virt.hardware [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 707.326030] env[62499]: DEBUG nova.virt.hardware [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 707.326218] env[62499]: DEBUG nova.virt.hardware [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 707.326388] env[62499]: DEBUG nova.virt.hardware [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 707.326564] env[62499]: DEBUG nova.virt.hardware [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 707.327712] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dfa8cc7c-00d7-40fe-b179-30df9114bf2f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 707.340418] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-87bb475f-be84-4bf4-9376-3348865c5819 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 707.357263] env[62499]: ERROR nova.compute.manager [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 2d6c29d5-c728-42bd-86a1-1f0309315094, please check neutron logs for more information. [ 707.357263] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Traceback (most recent call last): [ 707.357263] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 707.357263] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] yield resources [ 707.357263] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 707.357263] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] self.driver.spawn(context, instance, image_meta, [ 707.357263] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 707.357263] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 707.357263] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 707.357263] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] vm_ref = self.build_virtual_machine(instance, [ 707.357263] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 707.357673] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] vif_infos = vmwarevif.get_vif_info(self._session, [ 707.357673] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 707.357673] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] for vif in network_info: [ 707.357673] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 707.357673] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] return self._sync_wrapper(fn, *args, **kwargs) [ 707.357673] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 707.357673] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] self.wait() [ 707.357673] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 707.357673] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] self[:] = self._gt.wait() [ 707.357673] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 707.357673] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] return self._exit_event.wait() [ 707.357673] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 707.357673] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] current.throw(*self._exc) [ 707.358290] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 707.358290] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] result = function(*args, **kwargs) [ 707.358290] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 707.358290] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] return func(*args, **kwargs) [ 707.358290] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 707.358290] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] raise e [ 707.358290] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 707.358290] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] nwinfo = self.network_api.allocate_for_instance( [ 707.358290] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 707.358290] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] created_port_ids = self._update_ports_for_instance( [ 707.358290] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 707.358290] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] with excutils.save_and_reraise_exception(): [ 707.358290] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 707.358751] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] self.force_reraise() [ 707.358751] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 707.358751] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] raise self.value [ 707.358751] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 707.358751] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] updated_port = self._update_port( [ 707.358751] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 707.358751] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] _ensure_no_port_binding_failure(port) [ 707.358751] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 707.358751] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] raise exception.PortBindingFailed(port_id=port['id']) [ 707.358751] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] nova.exception.PortBindingFailed: Binding failed for port 2d6c29d5-c728-42bd-86a1-1f0309315094, please check neutron logs for more information. [ 707.358751] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] [ 707.358751] env[62499]: INFO nova.compute.manager [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Terminating instance [ 707.359647] env[62499]: DEBUG oslo_concurrency.lockutils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Acquiring lock "refresh_cache-3649431e-f856-4a28-9af7-8f101c3fc2d6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 707.667616] env[62499]: DEBUG nova.network.neutron [req-9c8839f9-d503-4250-a6c0-c6ecfc18e686 req-c4c25130-ad4e-4f9e-9df5-27481402b062 service nova] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 707.777761] env[62499]: DEBUG nova.network.neutron [req-9c8839f9-d503-4250-a6c0-c6ecfc18e686 req-c4c25130-ad4e-4f9e-9df5-27481402b062 service nova] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 707.805449] env[62499]: DEBUG oslo_concurrency.lockutils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.544s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 707.806246] env[62499]: DEBUG nova.compute.manager [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 707.809750] env[62499]: DEBUG oslo_concurrency.lockutils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.153s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 708.285642] env[62499]: DEBUG oslo_concurrency.lockutils [req-9c8839f9-d503-4250-a6c0-c6ecfc18e686 req-c4c25130-ad4e-4f9e-9df5-27481402b062 service nova] Releasing lock "refresh_cache-3649431e-f856-4a28-9af7-8f101c3fc2d6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 708.286084] env[62499]: DEBUG oslo_concurrency.lockutils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Acquired lock "refresh_cache-3649431e-f856-4a28-9af7-8f101c3fc2d6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 708.286270] env[62499]: DEBUG nova.network.neutron [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 708.314537] env[62499]: DEBUG nova.compute.utils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 708.319418] env[62499]: DEBUG nova.compute.manager [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 708.319849] env[62499]: DEBUG nova.network.neutron [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 708.361522] env[62499]: DEBUG nova.policy [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4762545dc5e14c16b5cbb1bad27d90a2', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3f11318c300c4c308f3f8a14db355954', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 708.694041] env[62499]: DEBUG nova.network.neutron [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Successfully created port: b10bfa9a-fa7b-425e-b992-d5db1489cc77 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 708.809845] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55804327-4ef1-445a-b7e4-4bb7b6797376 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 708.818771] env[62499]: DEBUG nova.compute.manager [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 708.822548] env[62499]: DEBUG nova.network.neutron [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 708.824935] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ce61b6f5-5a0b-4f06-b172-203499682b83 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 708.855482] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-73f86384-737e-4cc5-8710-b9c3f5834bd2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 708.863306] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e7db4e5-9a82-4ab4-8696-8e24ecd6e892 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 708.877622] env[62499]: DEBUG nova.compute.provider_tree [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 708.926308] env[62499]: DEBUG nova.network.neutron [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 709.250416] env[62499]: DEBUG nova.compute.manager [req-5b2db8d8-d512-44b3-9314-434ec293467b req-7e947c21-07ef-47ef-aa7b-b34441e01a8a service nova] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Received event network-vif-deleted-2d6c29d5-c728-42bd-86a1-1f0309315094 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 709.382301] env[62499]: DEBUG nova.scheduler.client.report [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 709.430387] env[62499]: DEBUG oslo_concurrency.lockutils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Releasing lock "refresh_cache-3649431e-f856-4a28-9af7-8f101c3fc2d6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 709.430387] env[62499]: DEBUG nova.compute.manager [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 709.430387] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 709.430387] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-045fe97e-a717-4ea2-bec4-a88bea82025b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 709.441408] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4a0e794f-29fc-4a48-998d-cf9231bb30b5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 709.463854] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3649431e-f856-4a28-9af7-8f101c3fc2d6 could not be found. [ 709.464111] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 709.464512] env[62499]: INFO nova.compute.manager [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Took 0.03 seconds to destroy the instance on the hypervisor. [ 709.464512] env[62499]: DEBUG oslo.service.loopingcall [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 709.464882] env[62499]: DEBUG nova.compute.manager [-] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 709.465038] env[62499]: DEBUG nova.network.neutron [-] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 709.479793] env[62499]: DEBUG nova.network.neutron [-] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 709.746665] env[62499]: ERROR nova.compute.manager [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b10bfa9a-fa7b-425e-b992-d5db1489cc77, please check neutron logs for more information. [ 709.746665] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 709.746665] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 709.746665] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 709.746665] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 709.746665] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 709.746665] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 709.746665] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 709.746665] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 709.746665] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 709.746665] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 709.746665] env[62499]: ERROR nova.compute.manager raise self.value [ 709.746665] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 709.746665] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 709.746665] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 709.746665] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 709.747241] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 709.747241] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 709.747241] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b10bfa9a-fa7b-425e-b992-d5db1489cc77, please check neutron logs for more information. [ 709.747241] env[62499]: ERROR nova.compute.manager [ 709.747241] env[62499]: Traceback (most recent call last): [ 709.747241] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 709.747241] env[62499]: listener.cb(fileno) [ 709.747241] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 709.747241] env[62499]: result = function(*args, **kwargs) [ 709.747241] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 709.747241] env[62499]: return func(*args, **kwargs) [ 709.747241] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 709.747241] env[62499]: raise e [ 709.747241] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 709.747241] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 709.747241] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 709.747241] env[62499]: created_port_ids = self._update_ports_for_instance( [ 709.747241] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 709.747241] env[62499]: with excutils.save_and_reraise_exception(): [ 709.747241] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 709.747241] env[62499]: self.force_reraise() [ 709.747241] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 709.747241] env[62499]: raise self.value [ 709.747241] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 709.747241] env[62499]: updated_port = self._update_port( [ 709.747241] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 709.747241] env[62499]: _ensure_no_port_binding_failure(port) [ 709.747241] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 709.747241] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 709.748188] env[62499]: nova.exception.PortBindingFailed: Binding failed for port b10bfa9a-fa7b-425e-b992-d5db1489cc77, please check neutron logs for more information. [ 709.748188] env[62499]: Removing descriptor: 16 [ 709.839754] env[62499]: DEBUG nova.compute.manager [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 709.872855] env[62499]: DEBUG nova.virt.hardware [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 709.872855] env[62499]: DEBUG nova.virt.hardware [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 709.872855] env[62499]: DEBUG nova.virt.hardware [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 709.873420] env[62499]: DEBUG nova.virt.hardware [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 709.877022] env[62499]: DEBUG nova.virt.hardware [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 709.877022] env[62499]: DEBUG nova.virt.hardware [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 709.877022] env[62499]: DEBUG nova.virt.hardware [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 709.877022] env[62499]: DEBUG nova.virt.hardware [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 709.877022] env[62499]: DEBUG nova.virt.hardware [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 709.877247] env[62499]: DEBUG nova.virt.hardware [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 709.877247] env[62499]: DEBUG nova.virt.hardware [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 709.877247] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f14a7f1a-7c88-4cfc-94d0-35a1489c184f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 709.888286] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e9d9c62-0f97-452d-aa73-e95541ee8882 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 709.893014] env[62499]: DEBUG oslo_concurrency.lockutils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.083s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 709.893014] env[62499]: ERROR nova.compute.manager [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 6a63349e-54c1-4d95-a74c-8515707dd186, please check neutron logs for more information. [ 709.893014] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Traceback (most recent call last): [ 709.893014] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 709.893014] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] self.driver.spawn(context, instance, image_meta, [ 709.893014] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 709.893014] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 709.893014] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 709.893014] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] vm_ref = self.build_virtual_machine(instance, [ 709.893447] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 709.893447] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] vif_infos = vmwarevif.get_vif_info(self._session, [ 709.893447] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 709.893447] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] for vif in network_info: [ 709.893447] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 709.893447] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] return self._sync_wrapper(fn, *args, **kwargs) [ 709.893447] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 709.893447] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] self.wait() [ 709.893447] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 709.893447] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] self[:] = self._gt.wait() [ 709.893447] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 709.893447] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] return self._exit_event.wait() [ 709.893447] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 709.893838] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] current.throw(*self._exc) [ 709.893838] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 709.893838] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] result = function(*args, **kwargs) [ 709.893838] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 709.893838] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] return func(*args, **kwargs) [ 709.893838] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 709.893838] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] raise e [ 709.893838] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 709.893838] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] nwinfo = self.network_api.allocate_for_instance( [ 709.893838] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 709.893838] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] created_port_ids = self._update_ports_for_instance( [ 709.893838] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 709.893838] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] with excutils.save_and_reraise_exception(): [ 709.894245] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 709.894245] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] self.force_reraise() [ 709.894245] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 709.894245] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] raise self.value [ 709.894245] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 709.894245] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] updated_port = self._update_port( [ 709.894245] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 709.894245] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] _ensure_no_port_binding_failure(port) [ 709.894245] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 709.894245] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] raise exception.PortBindingFailed(port_id=port['id']) [ 709.894245] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] nova.exception.PortBindingFailed: Binding failed for port 6a63349e-54c1-4d95-a74c-8515707dd186, please check neutron logs for more information. [ 709.894245] env[62499]: ERROR nova.compute.manager [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] [ 709.894616] env[62499]: DEBUG nova.compute.utils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Binding failed for port 6a63349e-54c1-4d95-a74c-8515707dd186, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 709.895061] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.524s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 709.896567] env[62499]: INFO nova.compute.claims [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 709.900747] env[62499]: DEBUG nova.compute.manager [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Build of instance 9ca13604-2b1b-4d47-8a23-8137384a87a6 was re-scheduled: Binding failed for port 6a63349e-54c1-4d95-a74c-8515707dd186, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 709.901208] env[62499]: DEBUG nova.compute.manager [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 709.901448] env[62499]: DEBUG oslo_concurrency.lockutils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Acquiring lock "refresh_cache-9ca13604-2b1b-4d47-8a23-8137384a87a6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 709.901633] env[62499]: DEBUG oslo_concurrency.lockutils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Acquired lock "refresh_cache-9ca13604-2b1b-4d47-8a23-8137384a87a6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 709.901829] env[62499]: DEBUG nova.network.neutron [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 709.912673] env[62499]: ERROR nova.compute.manager [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b10bfa9a-fa7b-425e-b992-d5db1489cc77, please check neutron logs for more information. [ 709.912673] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Traceback (most recent call last): [ 709.912673] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 709.912673] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] yield resources [ 709.912673] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 709.912673] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] self.driver.spawn(context, instance, image_meta, [ 709.912673] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 709.912673] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] self._vmops.spawn(context, instance, image_meta, injected_files, [ 709.912673] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 709.912673] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] vm_ref = self.build_virtual_machine(instance, [ 709.912673] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 709.913047] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] vif_infos = vmwarevif.get_vif_info(self._session, [ 709.913047] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 709.913047] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] for vif in network_info: [ 709.913047] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 709.913047] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] return self._sync_wrapper(fn, *args, **kwargs) [ 709.913047] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 709.913047] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] self.wait() [ 709.913047] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 709.913047] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] self[:] = self._gt.wait() [ 709.913047] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 709.913047] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] return self._exit_event.wait() [ 709.913047] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 709.913047] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] current.throw(*self._exc) [ 709.913395] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 709.913395] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] result = function(*args, **kwargs) [ 709.913395] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 709.913395] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] return func(*args, **kwargs) [ 709.913395] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 709.913395] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] raise e [ 709.913395] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 709.913395] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] nwinfo = self.network_api.allocate_for_instance( [ 709.913395] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 709.913395] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] created_port_ids = self._update_ports_for_instance( [ 709.913395] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 709.913395] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] with excutils.save_and_reraise_exception(): [ 709.913395] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 709.913723] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] self.force_reraise() [ 709.913723] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 709.913723] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] raise self.value [ 709.913723] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 709.913723] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] updated_port = self._update_port( [ 709.913723] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 709.913723] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] _ensure_no_port_binding_failure(port) [ 709.913723] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 709.913723] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] raise exception.PortBindingFailed(port_id=port['id']) [ 709.913723] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] nova.exception.PortBindingFailed: Binding failed for port b10bfa9a-fa7b-425e-b992-d5db1489cc77, please check neutron logs for more information. [ 709.913723] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] [ 709.913723] env[62499]: INFO nova.compute.manager [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Terminating instance [ 709.915591] env[62499]: DEBUG oslo_concurrency.lockutils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "refresh_cache-71fa08bc-a5c7-459c-b65b-5dacd9537281" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 709.915757] env[62499]: DEBUG oslo_concurrency.lockutils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquired lock "refresh_cache-71fa08bc-a5c7-459c-b65b-5dacd9537281" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 709.916043] env[62499]: DEBUG nova.network.neutron [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 709.982755] env[62499]: DEBUG nova.network.neutron [-] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 710.438923] env[62499]: DEBUG nova.network.neutron [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 710.444889] env[62499]: DEBUG nova.network.neutron [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 710.485054] env[62499]: INFO nova.compute.manager [-] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Took 1.02 seconds to deallocate network for instance. [ 710.487620] env[62499]: DEBUG nova.compute.claims [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 710.487787] env[62499]: DEBUG oslo_concurrency.lockutils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 710.546938] env[62499]: DEBUG nova.network.neutron [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 710.594240] env[62499]: DEBUG nova.network.neutron [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 711.053024] env[62499]: DEBUG oslo_concurrency.lockutils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Releasing lock "refresh_cache-9ca13604-2b1b-4d47-8a23-8137384a87a6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 711.053024] env[62499]: DEBUG nova.compute.manager [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 711.053024] env[62499]: DEBUG nova.compute.manager [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 711.053024] env[62499]: DEBUG nova.network.neutron [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 711.069646] env[62499]: DEBUG nova.network.neutron [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 711.096641] env[62499]: DEBUG oslo_concurrency.lockutils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Releasing lock "refresh_cache-71fa08bc-a5c7-459c-b65b-5dacd9537281" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 711.097075] env[62499]: DEBUG nova.compute.manager [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 711.097272] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 711.097843] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1cb9cf9d-5636-43fc-8f85-8a0eafe0548c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 711.108026] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e4a277d8-404a-4912-b146-c2276aa69d9b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 711.133864] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 71fa08bc-a5c7-459c-b65b-5dacd9537281 could not be found. [ 711.134144] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 711.134343] env[62499]: INFO nova.compute.manager [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Took 0.04 seconds to destroy the instance on the hypervisor. [ 711.134583] env[62499]: DEBUG oslo.service.loopingcall [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 711.137385] env[62499]: DEBUG nova.compute.manager [-] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 711.137781] env[62499]: DEBUG nova.network.neutron [-] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 711.163564] env[62499]: DEBUG nova.network.neutron [-] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 711.284688] env[62499]: DEBUG nova.compute.manager [req-873b42b8-095e-4acc-aa51-a9f3d29b3e09 req-f2a5d666-fdf2-40ec-96e7-0beef89308cb service nova] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Received event network-changed-b10bfa9a-fa7b-425e-b992-d5db1489cc77 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 711.284688] env[62499]: DEBUG nova.compute.manager [req-873b42b8-095e-4acc-aa51-a9f3d29b3e09 req-f2a5d666-fdf2-40ec-96e7-0beef89308cb service nova] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Refreshing instance network info cache due to event network-changed-b10bfa9a-fa7b-425e-b992-d5db1489cc77. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 711.284688] env[62499]: DEBUG oslo_concurrency.lockutils [req-873b42b8-095e-4acc-aa51-a9f3d29b3e09 req-f2a5d666-fdf2-40ec-96e7-0beef89308cb service nova] Acquiring lock "refresh_cache-71fa08bc-a5c7-459c-b65b-5dacd9537281" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 711.285174] env[62499]: DEBUG oslo_concurrency.lockutils [req-873b42b8-095e-4acc-aa51-a9f3d29b3e09 req-f2a5d666-fdf2-40ec-96e7-0beef89308cb service nova] Acquired lock "refresh_cache-71fa08bc-a5c7-459c-b65b-5dacd9537281" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 711.285174] env[62499]: DEBUG nova.network.neutron [req-873b42b8-095e-4acc-aa51-a9f3d29b3e09 req-f2a5d666-fdf2-40ec-96e7-0beef89308cb service nova] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Refreshing network info cache for port b10bfa9a-fa7b-425e-b992-d5db1489cc77 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 711.370872] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-57c45534-bc24-4891-a5ed-80919ec7ad49 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 711.381767] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2804e06-bb0b-44a0-9abc-ee85e85237e0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 711.412663] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7e92fc0-cabe-46a5-9f9b-3254b3a4de8e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 711.420815] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9c9276a-d9c4-49ce-86ce-05391555c65d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 711.433921] env[62499]: DEBUG nova.compute.provider_tree [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 711.574033] env[62499]: DEBUG nova.network.neutron [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 711.668690] env[62499]: DEBUG nova.network.neutron [-] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 711.805752] env[62499]: DEBUG nova.network.neutron [req-873b42b8-095e-4acc-aa51-a9f3d29b3e09 req-f2a5d666-fdf2-40ec-96e7-0beef89308cb service nova] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 711.917368] env[62499]: DEBUG nova.network.neutron [req-873b42b8-095e-4acc-aa51-a9f3d29b3e09 req-f2a5d666-fdf2-40ec-96e7-0beef89308cb service nova] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 711.937321] env[62499]: DEBUG nova.scheduler.client.report [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 712.078869] env[62499]: INFO nova.compute.manager [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] [instance: 9ca13604-2b1b-4d47-8a23-8137384a87a6] Took 1.03 seconds to deallocate network for instance. [ 712.172309] env[62499]: INFO nova.compute.manager [-] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Took 1.03 seconds to deallocate network for instance. [ 712.174731] env[62499]: DEBUG nova.compute.claims [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 712.176100] env[62499]: DEBUG oslo_concurrency.lockutils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 712.421108] env[62499]: DEBUG oslo_concurrency.lockutils [req-873b42b8-095e-4acc-aa51-a9f3d29b3e09 req-f2a5d666-fdf2-40ec-96e7-0beef89308cb service nova] Releasing lock "refresh_cache-71fa08bc-a5c7-459c-b65b-5dacd9537281" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 712.421108] env[62499]: DEBUG nova.compute.manager [req-873b42b8-095e-4acc-aa51-a9f3d29b3e09 req-f2a5d666-fdf2-40ec-96e7-0beef89308cb service nova] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Received event network-vif-deleted-b10bfa9a-fa7b-425e-b992-d5db1489cc77 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 712.444786] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.548s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 712.444786] env[62499]: DEBUG nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 712.446213] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.674s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 712.755685] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Acquiring lock "60a4b8c3-9dc4-4867-adca-4503e61ce237" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 712.755685] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Lock "60a4b8c3-9dc4-4867-adca-4503e61ce237" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 712.952436] env[62499]: DEBUG nova.compute.utils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 712.957696] env[62499]: DEBUG nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 712.957696] env[62499]: DEBUG nova.network.neutron [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 713.000693] env[62499]: DEBUG nova.policy [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '154e9fc2b68e4dc299ff4ff2419c9d33', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e176d3260a434fa2b4d2a113789c970a', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 713.114700] env[62499]: INFO nova.scheduler.client.report [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Deleted allocations for instance 9ca13604-2b1b-4d47-8a23-8137384a87a6 [ 713.205470] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Acquiring lock "ad749135-08c1-48ca-af56-300d6e796012" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 713.205705] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Lock "ad749135-08c1-48ca-af56-300d6e796012" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 713.457645] env[62499]: DEBUG nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 713.464776] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9097ad73-52f9-41a7-8161-9c0fbbde2ae7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 713.473207] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1ecc726d-5347-427e-a5ad-ee4c03e4feb9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 713.504511] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-296c89c1-bf3a-40f5-ab98-4e8b49642e5c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 713.513079] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8df7efd7-c794-4555-87ca-26f38c3ca7c7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 713.526992] env[62499]: DEBUG nova.compute.provider_tree [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 713.548502] env[62499]: DEBUG nova.network.neutron [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Successfully created port: 89eccc4e-8949-458d-a72c-cb35a3c3f973 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 713.626458] env[62499]: DEBUG oslo_concurrency.lockutils [None req-089bfebb-80bb-4d04-8f3a-fa0867aafd7e tempest-SecurityGroupsTestJSON-668110514 tempest-SecurityGroupsTestJSON-668110514-project-member] Lock "9ca13604-2b1b-4d47-8a23-8137384a87a6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 115.093s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 713.970066] env[62499]: INFO nova.virt.block_device [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Booting with volume ad811913-0e4c-42df-8394-485e010a5c02 at /dev/sda [ 714.009594] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d1b1e7ca-7283-4b86-ab05-33ad9cef596c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.018443] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95514eb2-3f30-4f26-afe4-466f88c0a022 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.029307] env[62499]: DEBUG nova.scheduler.client.report [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 714.044174] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f30b7e18-c0ed-4c48-b768-fd0f6d8092e3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.050804] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d69db76e-00de-4394-b07c-8cc624e9fb65 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.072767] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d74931fd-41cd-4d58-bc55-8319da6a5315 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.079054] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd0a03e4-7616-4df8-8fa4-2ffc5d0286cd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.092932] env[62499]: DEBUG nova.virt.block_device [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Updating existing volume attachment record: 0f38a77b-565c-4cd3-aac4-38a63267a8a1 {{(pid=62499) _volume_attach /opt/stack/nova/nova/virt/block_device.py:666}} [ 714.128949] env[62499]: DEBUG nova.compute.manager [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 714.535020] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.089s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 714.535691] env[62499]: ERROR nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 39337e32-ee39-47e5-ad79-389ade158ef2, please check neutron logs for more information. [ 714.535691] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Traceback (most recent call last): [ 714.535691] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 714.535691] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] self.driver.spawn(context, instance, image_meta, [ 714.535691] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 714.535691] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] self._vmops.spawn(context, instance, image_meta, injected_files, [ 714.535691] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 714.535691] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] vm_ref = self.build_virtual_machine(instance, [ 714.535691] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 714.535691] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] vif_infos = vmwarevif.get_vif_info(self._session, [ 714.535691] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 714.536109] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] for vif in network_info: [ 714.536109] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 714.536109] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] return self._sync_wrapper(fn, *args, **kwargs) [ 714.536109] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 714.536109] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] self.wait() [ 714.536109] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 714.536109] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] self[:] = self._gt.wait() [ 714.536109] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 714.536109] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] return self._exit_event.wait() [ 714.536109] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 714.536109] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] current.throw(*self._exc) [ 714.536109] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 714.536109] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] result = function(*args, **kwargs) [ 714.536509] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 714.536509] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] return func(*args, **kwargs) [ 714.536509] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 714.536509] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] raise e [ 714.536509] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 714.536509] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] nwinfo = self.network_api.allocate_for_instance( [ 714.536509] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 714.536509] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] created_port_ids = self._update_ports_for_instance( [ 714.536509] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 714.536509] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] with excutils.save_and_reraise_exception(): [ 714.536509] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 714.536509] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] self.force_reraise() [ 714.536509] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 714.536899] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] raise self.value [ 714.536899] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 714.536899] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] updated_port = self._update_port( [ 714.536899] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 714.536899] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] _ensure_no_port_binding_failure(port) [ 714.536899] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 714.536899] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] raise exception.PortBindingFailed(port_id=port['id']) [ 714.536899] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] nova.exception.PortBindingFailed: Binding failed for port 39337e32-ee39-47e5-ad79-389ade158ef2, please check neutron logs for more information. [ 714.536899] env[62499]: ERROR nova.compute.manager [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] [ 714.536899] env[62499]: DEBUG nova.compute.utils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Binding failed for port 39337e32-ee39-47e5-ad79-389ade158ef2, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 714.537802] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.706s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 714.539260] env[62499]: INFO nova.compute.claims [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 714.542329] env[62499]: DEBUG nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Build of instance 3b5fb42c-8a81-4575-ba8f-424a101d11ae was re-scheduled: Binding failed for port 39337e32-ee39-47e5-ad79-389ade158ef2, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 714.542745] env[62499]: DEBUG nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 714.542967] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Acquiring lock "refresh_cache-3b5fb42c-8a81-4575-ba8f-424a101d11ae" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 714.543124] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Acquired lock "refresh_cache-3b5fb42c-8a81-4575-ba8f-424a101d11ae" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 714.543282] env[62499]: DEBUG nova.network.neutron [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 714.553327] env[62499]: DEBUG nova.compute.manager [req-b28a3847-f29a-4756-9a69-9043d3e78036 req-641a1105-17ef-4b98-be75-2ae4d13470bb service nova] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Received event network-changed-89eccc4e-8949-458d-a72c-cb35a3c3f973 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 714.553510] env[62499]: DEBUG nova.compute.manager [req-b28a3847-f29a-4756-9a69-9043d3e78036 req-641a1105-17ef-4b98-be75-2ae4d13470bb service nova] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Refreshing instance network info cache due to event network-changed-89eccc4e-8949-458d-a72c-cb35a3c3f973. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 714.553710] env[62499]: DEBUG oslo_concurrency.lockutils [req-b28a3847-f29a-4756-9a69-9043d3e78036 req-641a1105-17ef-4b98-be75-2ae4d13470bb service nova] Acquiring lock "refresh_cache-6ae11b59-68af-482a-ad65-543a56d7aaa4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 714.553849] env[62499]: DEBUG oslo_concurrency.lockutils [req-b28a3847-f29a-4756-9a69-9043d3e78036 req-641a1105-17ef-4b98-be75-2ae4d13470bb service nova] Acquired lock "refresh_cache-6ae11b59-68af-482a-ad65-543a56d7aaa4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 714.554010] env[62499]: DEBUG nova.network.neutron [req-b28a3847-f29a-4756-9a69-9043d3e78036 req-641a1105-17ef-4b98-be75-2ae4d13470bb service nova] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Refreshing network info cache for port 89eccc4e-8949-458d-a72c-cb35a3c3f973 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 714.655632] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 714.758410] env[62499]: ERROR nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 89eccc4e-8949-458d-a72c-cb35a3c3f973, please check neutron logs for more information. [ 714.758410] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 714.758410] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 714.758410] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 714.758410] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 714.758410] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 714.758410] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 714.758410] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 714.758410] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 714.758410] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 714.758410] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 714.758410] env[62499]: ERROR nova.compute.manager raise self.value [ 714.758410] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 714.758410] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 714.758410] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 714.758410] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 714.758842] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 714.758842] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 714.758842] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 89eccc4e-8949-458d-a72c-cb35a3c3f973, please check neutron logs for more information. [ 714.758842] env[62499]: ERROR nova.compute.manager [ 714.758842] env[62499]: Traceback (most recent call last): [ 714.758842] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 714.758842] env[62499]: listener.cb(fileno) [ 714.758842] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 714.758842] env[62499]: result = function(*args, **kwargs) [ 714.758842] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 714.758842] env[62499]: return func(*args, **kwargs) [ 714.758842] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 714.758842] env[62499]: raise e [ 714.758842] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 714.758842] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 714.758842] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 714.758842] env[62499]: created_port_ids = self._update_ports_for_instance( [ 714.758842] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 714.758842] env[62499]: with excutils.save_and_reraise_exception(): [ 714.758842] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 714.758842] env[62499]: self.force_reraise() [ 714.758842] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 714.758842] env[62499]: raise self.value [ 714.758842] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 714.758842] env[62499]: updated_port = self._update_port( [ 714.758842] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 714.758842] env[62499]: _ensure_no_port_binding_failure(port) [ 714.758842] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 714.758842] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 714.760245] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 89eccc4e-8949-458d-a72c-cb35a3c3f973, please check neutron logs for more information. [ 714.760245] env[62499]: Removing descriptor: 17 [ 715.074791] env[62499]: DEBUG nova.network.neutron [req-b28a3847-f29a-4756-9a69-9043d3e78036 req-641a1105-17ef-4b98-be75-2ae4d13470bb service nova] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 715.079924] env[62499]: DEBUG nova.network.neutron [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 715.174080] env[62499]: DEBUG nova.network.neutron [req-b28a3847-f29a-4756-9a69-9043d3e78036 req-641a1105-17ef-4b98-be75-2ae4d13470bb service nova] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 715.207868] env[62499]: DEBUG nova.network.neutron [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 715.676067] env[62499]: DEBUG oslo_concurrency.lockutils [req-b28a3847-f29a-4756-9a69-9043d3e78036 req-641a1105-17ef-4b98-be75-2ae4d13470bb service nova] Releasing lock "refresh_cache-6ae11b59-68af-482a-ad65-543a56d7aaa4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 715.711202] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Releasing lock "refresh_cache-3b5fb42c-8a81-4575-ba8f-424a101d11ae" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 715.711437] env[62499]: DEBUG nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 715.711631] env[62499]: DEBUG nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 715.711804] env[62499]: DEBUG nova.network.neutron [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 715.737638] env[62499]: DEBUG nova.network.neutron [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 716.048245] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e81374f-d2bf-4c09-99d5-55ed6ee96abf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 716.056573] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b229252-1828-4a15-bb7d-125eb8a9da05 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 716.095523] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4a626c97-4af8-4eb2-8756-b66aa2a8313c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 716.104846] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44424a86-5f23-4494-8222-75313112f8ee {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 716.120074] env[62499]: DEBUG nova.compute.provider_tree [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 716.215140] env[62499]: DEBUG nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 716.215756] env[62499]: DEBUG nova.virt.hardware [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum=,container_format=,created_at=,direct_url=,disk_format=,id=,min_disk=0,min_ram=0,name=,owner=,properties=ImageMetaProps,protected=,size=1073741824,status='active',tags=,updated_at=,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 716.215971] env[62499]: DEBUG nova.virt.hardware [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 716.216141] env[62499]: DEBUG nova.virt.hardware [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 716.216323] env[62499]: DEBUG nova.virt.hardware [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 716.216469] env[62499]: DEBUG nova.virt.hardware [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 716.216626] env[62499]: DEBUG nova.virt.hardware [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 716.216821] env[62499]: DEBUG nova.virt.hardware [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 716.216979] env[62499]: DEBUG nova.virt.hardware [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 716.217192] env[62499]: DEBUG nova.virt.hardware [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 716.217406] env[62499]: DEBUG nova.virt.hardware [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 716.217586] env[62499]: DEBUG nova.virt.hardware [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 716.218468] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27b45574-f81d-45b6-b737-47b03782dd50 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 716.227013] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85438ea0-1f38-486e-9057-cfce94d77cd1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 716.240887] env[62499]: DEBUG nova.network.neutron [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 716.242746] env[62499]: ERROR nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 89eccc4e-8949-458d-a72c-cb35a3c3f973, please check neutron logs for more information. [ 716.242746] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Traceback (most recent call last): [ 716.242746] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 716.242746] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] yield resources [ 716.242746] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 716.242746] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] self.driver.spawn(context, instance, image_meta, [ 716.242746] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 716.242746] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 716.242746] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 716.242746] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] vm_ref = self.build_virtual_machine(instance, [ 716.242746] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 716.243071] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] vif_infos = vmwarevif.get_vif_info(self._session, [ 716.243071] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 716.243071] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] for vif in network_info: [ 716.243071] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 716.243071] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] return self._sync_wrapper(fn, *args, **kwargs) [ 716.243071] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 716.243071] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] self.wait() [ 716.243071] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 716.243071] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] self[:] = self._gt.wait() [ 716.243071] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 716.243071] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] return self._exit_event.wait() [ 716.243071] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 716.243071] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] current.throw(*self._exc) [ 716.243784] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 716.243784] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] result = function(*args, **kwargs) [ 716.243784] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 716.243784] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] return func(*args, **kwargs) [ 716.243784] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 716.243784] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] raise e [ 716.243784] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 716.243784] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] nwinfo = self.network_api.allocate_for_instance( [ 716.243784] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 716.243784] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] created_port_ids = self._update_ports_for_instance( [ 716.243784] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 716.243784] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] with excutils.save_and_reraise_exception(): [ 716.243784] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 716.244263] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] self.force_reraise() [ 716.244263] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 716.244263] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] raise self.value [ 716.244263] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 716.244263] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] updated_port = self._update_port( [ 716.244263] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 716.244263] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] _ensure_no_port_binding_failure(port) [ 716.244263] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 716.244263] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] raise exception.PortBindingFailed(port_id=port['id']) [ 716.244263] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] nova.exception.PortBindingFailed: Binding failed for port 89eccc4e-8949-458d-a72c-cb35a3c3f973, please check neutron logs for more information. [ 716.244263] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] [ 716.244263] env[62499]: INFO nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Terminating instance [ 716.244780] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Acquiring lock "refresh_cache-6ae11b59-68af-482a-ad65-543a56d7aaa4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 716.244937] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Acquired lock "refresh_cache-6ae11b59-68af-482a-ad65-543a56d7aaa4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 716.245117] env[62499]: DEBUG nova.network.neutron [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 716.581485] env[62499]: DEBUG nova.compute.manager [req-e7e82a80-e066-446c-af2b-5a8cdd2f2669 req-a7eceaf9-4de9-462d-8e2b-026ed6cce9f4 service nova] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Received event network-vif-deleted-89eccc4e-8949-458d-a72c-cb35a3c3f973 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 716.623638] env[62499]: DEBUG nova.scheduler.client.report [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 716.746682] env[62499]: INFO nova.compute.manager [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] [instance: 3b5fb42c-8a81-4575-ba8f-424a101d11ae] Took 1.03 seconds to deallocate network for instance. [ 716.775200] env[62499]: DEBUG nova.network.neutron [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 716.902991] env[62499]: DEBUG nova.network.neutron [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 717.129020] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.591s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 717.129576] env[62499]: DEBUG nova.compute.manager [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 717.132622] env[62499]: DEBUG oslo_concurrency.lockutils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.179s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 717.406477] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Releasing lock "refresh_cache-6ae11b59-68af-482a-ad65-543a56d7aaa4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 717.407070] env[62499]: DEBUG nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 717.407399] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4406fb48-96b1-48ca-889c-940605a9ebe7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 717.416638] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8a38d7e4-b77c-4b18-a087-750627a66723 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 717.440166] env[62499]: WARNING nova.virt.vmwareapi.driver [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Instance does not exists. Proceeding to delete instance properties on datastore: nova.exception.InstanceNotFound: Instance 6ae11b59-68af-482a-ad65-543a56d7aaa4 could not be found. [ 717.441127] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 717.441127] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-27de9b47-9a9c-4c28-bcda-be903c61ddbf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 717.448538] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-15df2c27-81c4-45a5-b71e-e3dec4b540bd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 717.470475] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6ae11b59-68af-482a-ad65-543a56d7aaa4 could not be found. [ 717.470475] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 717.470664] env[62499]: INFO nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Took 0.06 seconds to destroy the instance on the hypervisor. [ 717.470825] env[62499]: DEBUG oslo.service.loopingcall [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 717.470960] env[62499]: DEBUG nova.compute.manager [-] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 717.471068] env[62499]: DEBUG nova.network.neutron [-] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 717.488250] env[62499]: DEBUG nova.network.neutron [-] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 717.637909] env[62499]: DEBUG nova.compute.utils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 717.643886] env[62499]: DEBUG nova.compute.manager [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 717.645367] env[62499]: DEBUG nova.network.neutron [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 717.704112] env[62499]: DEBUG nova.policy [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '5a4553a28ba342a98948136b4d24874a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5fe57bb058af414d9a696248dd14fa7b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 717.785969] env[62499]: INFO nova.scheduler.client.report [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Deleted allocations for instance 3b5fb42c-8a81-4575-ba8f-424a101d11ae [ 717.990798] env[62499]: DEBUG nova.network.neutron [-] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 718.082623] env[62499]: DEBUG nova.network.neutron [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Successfully created port: d4db130b-8be3-4540-8b5d-cec6c5fc0da5 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 718.118809] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f2d1922-f506-4749-958c-e0607101809f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 718.127637] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ecedbb1-c78e-4779-ba13-c2479fe1b8f2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 718.162014] env[62499]: DEBUG nova.compute.manager [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 718.165478] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bcb88bf0-bb69-445d-a507-bf12d0c9ea2a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 718.175056] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed962a72-b917-406d-b7e9-d7014274bc65 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 718.189113] env[62499]: DEBUG nova.compute.provider_tree [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 718.297512] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e789abf5-25c9-45bc-8416-5658e6c30b38 tempest-ServersTestBootFromVolume-1640696889 tempest-ServersTestBootFromVolume-1640696889-project-member] Lock "3b5fb42c-8a81-4575-ba8f-424a101d11ae" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 119.363s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 718.494667] env[62499]: INFO nova.compute.manager [-] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Took 1.02 seconds to deallocate network for instance. [ 718.693369] env[62499]: DEBUG nova.scheduler.client.report [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 718.802298] env[62499]: DEBUG nova.compute.manager [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 719.041144] env[62499]: DEBUG nova.compute.manager [req-3a712b41-d253-4b1e-bbf2-e525ff48ae72 req-1ca3ca06-448d-4d3b-9d9c-612d171884b6 service nova] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Received event network-changed-d4db130b-8be3-4540-8b5d-cec6c5fc0da5 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 719.041144] env[62499]: DEBUG nova.compute.manager [req-3a712b41-d253-4b1e-bbf2-e525ff48ae72 req-1ca3ca06-448d-4d3b-9d9c-612d171884b6 service nova] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Refreshing instance network info cache due to event network-changed-d4db130b-8be3-4540-8b5d-cec6c5fc0da5. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 719.041144] env[62499]: DEBUG oslo_concurrency.lockutils [req-3a712b41-d253-4b1e-bbf2-e525ff48ae72 req-1ca3ca06-448d-4d3b-9d9c-612d171884b6 service nova] Acquiring lock "refresh_cache-6d8b7855-567c-44c3-90ee-fc0daec87f46" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 719.041144] env[62499]: DEBUG oslo_concurrency.lockutils [req-3a712b41-d253-4b1e-bbf2-e525ff48ae72 req-1ca3ca06-448d-4d3b-9d9c-612d171884b6 service nova] Acquired lock "refresh_cache-6d8b7855-567c-44c3-90ee-fc0daec87f46" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 719.041144] env[62499]: DEBUG nova.network.neutron [req-3a712b41-d253-4b1e-bbf2-e525ff48ae72 req-1ca3ca06-448d-4d3b-9d9c-612d171884b6 service nova] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Refreshing network info cache for port d4db130b-8be3-4540-8b5d-cec6c5fc0da5 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 719.073508] env[62499]: INFO nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Took 0.58 seconds to detach 1 volumes for instance. [ 719.075836] env[62499]: DEBUG nova.compute.claims [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 719.076634] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 719.181506] env[62499]: DEBUG nova.compute.manager [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 719.201085] env[62499]: DEBUG oslo_concurrency.lockutils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.068s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 719.201708] env[62499]: ERROR nova.compute.manager [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ddd7b2bb-0a7c-4f1f-ad88-162c46df78b1, please check neutron logs for more information. [ 719.201708] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] Traceback (most recent call last): [ 719.201708] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 719.201708] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] self.driver.spawn(context, instance, image_meta, [ 719.201708] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 719.201708] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 719.201708] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 719.201708] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] vm_ref = self.build_virtual_machine(instance, [ 719.201708] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 719.201708] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] vif_infos = vmwarevif.get_vif_info(self._session, [ 719.201708] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 719.202081] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] for vif in network_info: [ 719.202081] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 719.202081] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] return self._sync_wrapper(fn, *args, **kwargs) [ 719.202081] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 719.202081] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] self.wait() [ 719.202081] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 719.202081] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] self[:] = self._gt.wait() [ 719.202081] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 719.202081] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] return self._exit_event.wait() [ 719.202081] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 719.202081] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] current.throw(*self._exc) [ 719.202081] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 719.202081] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] result = function(*args, **kwargs) [ 719.203323] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 719.203323] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] return func(*args, **kwargs) [ 719.203323] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 719.203323] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] raise e [ 719.203323] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 719.203323] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] nwinfo = self.network_api.allocate_for_instance( [ 719.203323] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 719.203323] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] created_port_ids = self._update_ports_for_instance( [ 719.203323] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 719.203323] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] with excutils.save_and_reraise_exception(): [ 719.203323] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 719.203323] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] self.force_reraise() [ 719.203323] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 719.203751] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] raise self.value [ 719.203751] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 719.203751] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] updated_port = self._update_port( [ 719.203751] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 719.203751] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] _ensure_no_port_binding_failure(port) [ 719.203751] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 719.203751] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] raise exception.PortBindingFailed(port_id=port['id']) [ 719.203751] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] nova.exception.PortBindingFailed: Binding failed for port ddd7b2bb-0a7c-4f1f-ad88-162c46df78b1, please check neutron logs for more information. [ 719.203751] env[62499]: ERROR nova.compute.manager [instance: 0899249c-880c-48de-b2c3-2c407115463e] [ 719.203751] env[62499]: DEBUG nova.compute.utils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Binding failed for port ddd7b2bb-0a7c-4f1f-ad88-162c46df78b1, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 719.205503] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.724s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 719.212023] env[62499]: DEBUG nova.compute.manager [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Build of instance 0899249c-880c-48de-b2c3-2c407115463e was re-scheduled: Binding failed for port ddd7b2bb-0a7c-4f1f-ad88-162c46df78b1, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 719.212023] env[62499]: DEBUG nova.compute.manager [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 719.212023] env[62499]: DEBUG oslo_concurrency.lockutils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Acquiring lock "refresh_cache-0899249c-880c-48de-b2c3-2c407115463e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 719.212023] env[62499]: DEBUG oslo_concurrency.lockutils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Acquired lock "refresh_cache-0899249c-880c-48de-b2c3-2c407115463e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 719.212556] env[62499]: DEBUG nova.network.neutron [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 719.218388] env[62499]: DEBUG nova.virt.hardware [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 719.218607] env[62499]: DEBUG nova.virt.hardware [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 719.218761] env[62499]: DEBUG nova.virt.hardware [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 719.218933] env[62499]: DEBUG nova.virt.hardware [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 719.219084] env[62499]: DEBUG nova.virt.hardware [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 719.219230] env[62499]: DEBUG nova.virt.hardware [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 719.219465] env[62499]: DEBUG nova.virt.hardware [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 719.219624] env[62499]: DEBUG nova.virt.hardware [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 719.220424] env[62499]: DEBUG nova.virt.hardware [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 719.220424] env[62499]: DEBUG nova.virt.hardware [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 719.222023] env[62499]: DEBUG nova.virt.hardware [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 719.222122] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff1a2a39-830f-466e-be85-3fdab0cf3cca {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 719.232199] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-31e01c35-15f6-488c-9052-41be267c42a2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 719.332695] env[62499]: DEBUG oslo_concurrency.lockutils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 719.500212] env[62499]: ERROR nova.compute.manager [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d4db130b-8be3-4540-8b5d-cec6c5fc0da5, please check neutron logs for more information. [ 719.500212] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 719.500212] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 719.500212] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 719.500212] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 719.500212] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 719.500212] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 719.500212] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 719.500212] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 719.500212] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 719.500212] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 719.500212] env[62499]: ERROR nova.compute.manager raise self.value [ 719.500212] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 719.500212] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 719.500212] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 719.500212] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 719.500807] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 719.500807] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 719.500807] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d4db130b-8be3-4540-8b5d-cec6c5fc0da5, please check neutron logs for more information. [ 719.500807] env[62499]: ERROR nova.compute.manager [ 719.500807] env[62499]: Traceback (most recent call last): [ 719.500807] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 719.500807] env[62499]: listener.cb(fileno) [ 719.500807] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 719.500807] env[62499]: result = function(*args, **kwargs) [ 719.500807] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 719.500807] env[62499]: return func(*args, **kwargs) [ 719.500807] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 719.500807] env[62499]: raise e [ 719.500807] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 719.500807] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 719.500807] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 719.500807] env[62499]: created_port_ids = self._update_ports_for_instance( [ 719.500807] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 719.500807] env[62499]: with excutils.save_and_reraise_exception(): [ 719.500807] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 719.500807] env[62499]: self.force_reraise() [ 719.500807] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 719.500807] env[62499]: raise self.value [ 719.500807] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 719.500807] env[62499]: updated_port = self._update_port( [ 719.500807] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 719.500807] env[62499]: _ensure_no_port_binding_failure(port) [ 719.500807] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 719.500807] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 719.501593] env[62499]: nova.exception.PortBindingFailed: Binding failed for port d4db130b-8be3-4540-8b5d-cec6c5fc0da5, please check neutron logs for more information. [ 719.501593] env[62499]: Removing descriptor: 17 [ 719.501593] env[62499]: ERROR nova.compute.manager [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d4db130b-8be3-4540-8b5d-cec6c5fc0da5, please check neutron logs for more information. [ 719.501593] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Traceback (most recent call last): [ 719.501593] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 719.501593] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] yield resources [ 719.501593] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 719.501593] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] self.driver.spawn(context, instance, image_meta, [ 719.501593] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 719.501593] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] self._vmops.spawn(context, instance, image_meta, injected_files, [ 719.501593] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 719.501593] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] vm_ref = self.build_virtual_machine(instance, [ 719.501911] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 719.501911] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] vif_infos = vmwarevif.get_vif_info(self._session, [ 719.501911] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 719.501911] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] for vif in network_info: [ 719.501911] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 719.501911] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] return self._sync_wrapper(fn, *args, **kwargs) [ 719.501911] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 719.501911] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] self.wait() [ 719.501911] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 719.501911] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] self[:] = self._gt.wait() [ 719.501911] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 719.501911] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] return self._exit_event.wait() [ 719.501911] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 719.502313] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] result = hub.switch() [ 719.502313] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 719.502313] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] return self.greenlet.switch() [ 719.502313] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 719.502313] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] result = function(*args, **kwargs) [ 719.502313] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 719.502313] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] return func(*args, **kwargs) [ 719.502313] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 719.502313] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] raise e [ 719.502313] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 719.502313] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] nwinfo = self.network_api.allocate_for_instance( [ 719.502313] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 719.502313] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] created_port_ids = self._update_ports_for_instance( [ 719.502646] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 719.502646] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] with excutils.save_and_reraise_exception(): [ 719.502646] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 719.502646] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] self.force_reraise() [ 719.502646] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 719.502646] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] raise self.value [ 719.502646] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 719.502646] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] updated_port = self._update_port( [ 719.502646] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 719.502646] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] _ensure_no_port_binding_failure(port) [ 719.502646] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 719.502646] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] raise exception.PortBindingFailed(port_id=port['id']) [ 719.503031] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] nova.exception.PortBindingFailed: Binding failed for port d4db130b-8be3-4540-8b5d-cec6c5fc0da5, please check neutron logs for more information. [ 719.503031] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] [ 719.503031] env[62499]: INFO nova.compute.manager [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Terminating instance [ 719.505391] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Acquiring lock "refresh_cache-6d8b7855-567c-44c3-90ee-fc0daec87f46" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 719.566536] env[62499]: DEBUG nova.network.neutron [req-3a712b41-d253-4b1e-bbf2-e525ff48ae72 req-1ca3ca06-448d-4d3b-9d9c-612d171884b6 service nova] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 719.651642] env[62499]: DEBUG nova.network.neutron [req-3a712b41-d253-4b1e-bbf2-e525ff48ae72 req-1ca3ca06-448d-4d3b-9d9c-612d171884b6 service nova] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 719.744829] env[62499]: DEBUG nova.network.neutron [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 719.910619] env[62499]: DEBUG nova.network.neutron [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 720.157647] env[62499]: DEBUG oslo_concurrency.lockutils [req-3a712b41-d253-4b1e-bbf2-e525ff48ae72 req-1ca3ca06-448d-4d3b-9d9c-612d171884b6 service nova] Releasing lock "refresh_cache-6d8b7855-567c-44c3-90ee-fc0daec87f46" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 720.158099] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Acquired lock "refresh_cache-6d8b7855-567c-44c3-90ee-fc0daec87f46" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 720.158289] env[62499]: DEBUG nova.network.neutron [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 720.301460] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5c56b2a7-2061-4eee-8115-28a8392aa4b9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 720.309196] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-316b44a7-83d5-465f-bec3-b34c0ca90ad2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 720.339730] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09cef2ec-44e5-4a49-b2c7-835d43c30b4a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 720.347069] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0110763a-12fc-4248-a13a-8f9643632a59 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 720.363636] env[62499]: DEBUG nova.compute.provider_tree [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 720.413950] env[62499]: DEBUG oslo_concurrency.lockutils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Releasing lock "refresh_cache-0899249c-880c-48de-b2c3-2c407115463e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 720.414203] env[62499]: DEBUG nova.compute.manager [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 720.414392] env[62499]: DEBUG nova.compute.manager [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 720.414551] env[62499]: DEBUG nova.network.neutron [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 720.430838] env[62499]: DEBUG nova.network.neutron [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 720.687264] env[62499]: DEBUG nova.network.neutron [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 720.829660] env[62499]: DEBUG nova.network.neutron [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 720.868730] env[62499]: DEBUG nova.scheduler.client.report [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 720.934661] env[62499]: DEBUG nova.network.neutron [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 721.072742] env[62499]: DEBUG nova.compute.manager [req-e7e565c0-fb11-4a74-8a66-c2a1365e9a9e req-9180576a-7a01-48f9-bf1a-fae1c7c9036d service nova] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Received event network-vif-deleted-d4db130b-8be3-4540-8b5d-cec6c5fc0da5 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 721.333484] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Releasing lock "refresh_cache-6d8b7855-567c-44c3-90ee-fc0daec87f46" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 721.333959] env[62499]: DEBUG nova.compute.manager [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 721.334179] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 721.334809] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2904e5b5-b845-4756-9daf-de6a9987fa5e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 721.351556] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c8d487a-21c2-4c61-87dd-a0985bd47956 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 721.377384] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.172s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 721.378016] env[62499]: ERROR nova.compute.manager [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port da8078a3-f87c-46cd-ad54-61d196be63f0, please check neutron logs for more information. [ 721.378016] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Traceback (most recent call last): [ 721.378016] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 721.378016] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] self.driver.spawn(context, instance, image_meta, [ 721.378016] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 721.378016] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 721.378016] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 721.378016] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] vm_ref = self.build_virtual_machine(instance, [ 721.378016] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 721.378016] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] vif_infos = vmwarevif.get_vif_info(self._session, [ 721.378016] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 721.378347] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] for vif in network_info: [ 721.378347] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 721.378347] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] return self._sync_wrapper(fn, *args, **kwargs) [ 721.378347] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 721.378347] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] self.wait() [ 721.378347] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 721.378347] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] self[:] = self._gt.wait() [ 721.378347] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 721.378347] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] return self._exit_event.wait() [ 721.378347] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 721.378347] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] current.throw(*self._exc) [ 721.378347] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 721.378347] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] result = function(*args, **kwargs) [ 721.378705] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 721.378705] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] return func(*args, **kwargs) [ 721.378705] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 721.378705] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] raise e [ 721.378705] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 721.378705] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] nwinfo = self.network_api.allocate_for_instance( [ 721.378705] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 721.378705] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] created_port_ids = self._update_ports_for_instance( [ 721.378705] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 721.378705] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] with excutils.save_and_reraise_exception(): [ 721.378705] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 721.378705] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] self.force_reraise() [ 721.378705] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 721.379108] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] raise self.value [ 721.379108] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 721.379108] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] updated_port = self._update_port( [ 721.379108] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 721.379108] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] _ensure_no_port_binding_failure(port) [ 721.379108] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 721.379108] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] raise exception.PortBindingFailed(port_id=port['id']) [ 721.379108] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] nova.exception.PortBindingFailed: Binding failed for port da8078a3-f87c-46cd-ad54-61d196be63f0, please check neutron logs for more information. [ 721.379108] env[62499]: ERROR nova.compute.manager [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] [ 721.379108] env[62499]: DEBUG nova.compute.utils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Binding failed for port da8078a3-f87c-46cd-ad54-61d196be63f0, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 721.380207] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6d8b7855-567c-44c3-90ee-fc0daec87f46 could not be found. [ 721.380408] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 721.380583] env[62499]: INFO nova.compute.manager [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Took 0.05 seconds to destroy the instance on the hypervisor. [ 721.381015] env[62499]: DEBUG oslo.service.loopingcall [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 721.381870] env[62499]: DEBUG nova.compute.manager [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Build of instance a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b was re-scheduled: Binding failed for port da8078a3-f87c-46cd-ad54-61d196be63f0, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 721.381870] env[62499]: DEBUG nova.compute.manager [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 721.382248] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Acquiring lock "refresh_cache-a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 721.382421] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Acquired lock "refresh_cache-a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 721.383034] env[62499]: DEBUG nova.network.neutron [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 721.383519] env[62499]: DEBUG oslo_concurrency.lockutils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.364s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 721.387353] env[62499]: DEBUG nova.compute.manager [-] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 721.387353] env[62499]: DEBUG nova.network.neutron [-] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 721.408122] env[62499]: DEBUG nova.network.neutron [-] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 721.436477] env[62499]: INFO nova.compute.manager [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] [instance: 0899249c-880c-48de-b2c3-2c407115463e] Took 1.02 seconds to deallocate network for instance. [ 721.910579] env[62499]: DEBUG nova.network.neutron [-] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 721.923029] env[62499]: DEBUG nova.network.neutron [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 722.038797] env[62499]: DEBUG nova.network.neutron [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 722.251110] env[62499]: DEBUG oslo_concurrency.lockutils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Acquiring lock "2a1c52ab-8ee6-4b63-ad04-2450a60262d0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 722.251549] env[62499]: DEBUG oslo_concurrency.lockutils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Lock "2a1c52ab-8ee6-4b63-ad04-2450a60262d0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 722.368968] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6eafef4d-a424-473c-9741-d0ad98adf1c7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 722.376725] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2c92dda6-9e82-4b51-8d64-9e6b97812972 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 722.405675] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52b1dd82-4e85-46a2-8dea-a1d283b55eca {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 722.413016] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-26745f7d-c127-4543-bc4b-524afb461b1e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 722.416984] env[62499]: INFO nova.compute.manager [-] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Took 1.03 seconds to deallocate network for instance. [ 722.419488] env[62499]: DEBUG nova.compute.claims [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 722.419674] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 722.427780] env[62499]: DEBUG nova.compute.provider_tree [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 722.469648] env[62499]: INFO nova.scheduler.client.report [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Deleted allocations for instance 0899249c-880c-48de-b2c3-2c407115463e [ 722.541902] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Releasing lock "refresh_cache-a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 722.543609] env[62499]: DEBUG nova.compute.manager [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 722.543609] env[62499]: DEBUG nova.compute.manager [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 722.543609] env[62499]: DEBUG nova.network.neutron [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 722.556806] env[62499]: DEBUG nova.network.neutron [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 722.932777] env[62499]: DEBUG nova.scheduler.client.report [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 722.978557] env[62499]: DEBUG oslo_concurrency.lockutils [None req-791139e3-6020-4fd8-9217-e882bd37d858 tempest-InstanceActionsNegativeTestJSON-1284918533 tempest-InstanceActionsNegativeTestJSON-1284918533-project-member] Lock "0899249c-880c-48de-b2c3-2c407115463e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 122.364s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 723.059529] env[62499]: DEBUG nova.network.neutron [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 723.437637] env[62499]: DEBUG oslo_concurrency.lockutils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.054s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 723.438245] env[62499]: ERROR nova.compute.manager [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 993e00ef-a852-4091-bc3a-7acc78478536, please check neutron logs for more information. [ 723.438245] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Traceback (most recent call last): [ 723.438245] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 723.438245] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] self.driver.spawn(context, instance, image_meta, [ 723.438245] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 723.438245] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] self._vmops.spawn(context, instance, image_meta, injected_files, [ 723.438245] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 723.438245] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] vm_ref = self.build_virtual_machine(instance, [ 723.438245] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 723.438245] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] vif_infos = vmwarevif.get_vif_info(self._session, [ 723.438245] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 723.438628] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] for vif in network_info: [ 723.438628] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 723.438628] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] return self._sync_wrapper(fn, *args, **kwargs) [ 723.438628] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 723.438628] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] self.wait() [ 723.438628] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 723.438628] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] self[:] = self._gt.wait() [ 723.438628] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 723.438628] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] return self._exit_event.wait() [ 723.438628] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 723.438628] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] current.throw(*self._exc) [ 723.438628] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 723.438628] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] result = function(*args, **kwargs) [ 723.439030] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 723.439030] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] return func(*args, **kwargs) [ 723.439030] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 723.439030] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] raise e [ 723.439030] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 723.439030] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] nwinfo = self.network_api.allocate_for_instance( [ 723.439030] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 723.439030] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] created_port_ids = self._update_ports_for_instance( [ 723.439030] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 723.439030] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] with excutils.save_and_reraise_exception(): [ 723.439030] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 723.439030] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] self.force_reraise() [ 723.439030] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 723.439444] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] raise self.value [ 723.439444] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 723.439444] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] updated_port = self._update_port( [ 723.439444] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 723.439444] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] _ensure_no_port_binding_failure(port) [ 723.439444] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 723.439444] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] raise exception.PortBindingFailed(port_id=port['id']) [ 723.439444] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] nova.exception.PortBindingFailed: Binding failed for port 993e00ef-a852-4091-bc3a-7acc78478536, please check neutron logs for more information. [ 723.439444] env[62499]: ERROR nova.compute.manager [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] [ 723.439444] env[62499]: DEBUG nova.compute.utils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Binding failed for port 993e00ef-a852-4091-bc3a-7acc78478536, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 723.440388] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.421s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 723.441696] env[62499]: INFO nova.compute.claims [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 723.444196] env[62499]: DEBUG nova.compute.manager [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Build of instance 3d9ac347-ffcc-4840-8090-811dbd2865ef was re-scheduled: Binding failed for port 993e00ef-a852-4091-bc3a-7acc78478536, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 723.444604] env[62499]: DEBUG nova.compute.manager [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 723.444828] env[62499]: DEBUG oslo_concurrency.lockutils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquiring lock "refresh_cache-3d9ac347-ffcc-4840-8090-811dbd2865ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 723.444969] env[62499]: DEBUG oslo_concurrency.lockutils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquired lock "refresh_cache-3d9ac347-ffcc-4840-8090-811dbd2865ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 723.445138] env[62499]: DEBUG nova.network.neutron [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 723.481378] env[62499]: DEBUG nova.compute.manager [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 723.563396] env[62499]: INFO nova.compute.manager [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] [instance: a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b] Took 1.02 seconds to deallocate network for instance. [ 723.976789] env[62499]: DEBUG nova.network.neutron [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 724.006241] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 724.144060] env[62499]: DEBUG nova.network.neutron [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 724.602603] env[62499]: INFO nova.scheduler.client.report [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Deleted allocations for instance a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b [ 724.650478] env[62499]: DEBUG oslo_concurrency.lockutils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Releasing lock "refresh_cache-3d9ac347-ffcc-4840-8090-811dbd2865ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 724.651535] env[62499]: DEBUG nova.compute.manager [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 724.651535] env[62499]: DEBUG nova.compute.manager [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 724.651535] env[62499]: DEBUG nova.network.neutron [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 724.674932] env[62499]: DEBUG nova.network.neutron [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 724.894412] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-edc85b4e-4fdd-42fa-94e7-8c29b35b6b9b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 724.902835] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f32e909-86d8-499b-8dfc-f3f3a1a14a28 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 724.935355] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99cc20bf-104b-4959-b8da-ad30a13ec14e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 724.943439] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5933f522-e753-4f31-8ea6-658467f7e4bc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 724.957229] env[62499]: DEBUG nova.compute.provider_tree [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 725.116424] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b0c3ddd2-2aa9-4fe5-9b0d-baaff0458097 tempest-ImagesOneServerNegativeTestJSON-1329507913 tempest-ImagesOneServerNegativeTestJSON-1329507913-project-member] Lock "a4a9c3d1-70f5-4521-9325-e3a7f6e7de1b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 124.421s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 725.179502] env[62499]: DEBUG nova.network.neutron [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 725.460927] env[62499]: DEBUG nova.scheduler.client.report [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 725.621546] env[62499]: DEBUG nova.compute.manager [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 725.684646] env[62499]: INFO nova.compute.manager [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 3d9ac347-ffcc-4840-8090-811dbd2865ef] Took 1.03 seconds to deallocate network for instance. [ 725.968585] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.528s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 725.969136] env[62499]: DEBUG nova.compute.manager [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 725.976541] env[62499]: DEBUG oslo_concurrency.lockutils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.486s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 726.151026] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 726.480194] env[62499]: DEBUG nova.compute.utils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 726.484506] env[62499]: DEBUG nova.compute.manager [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 726.484682] env[62499]: DEBUG nova.network.neutron [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 726.527398] env[62499]: DEBUG nova.policy [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '70fa403562d44a82b91063fd0b4a2664', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5bb308bb00f048f4bd47b3cd77843a58', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 726.717725] env[62499]: INFO nova.scheduler.client.report [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Deleted allocations for instance 3d9ac347-ffcc-4840-8090-811dbd2865ef [ 726.866633] env[62499]: DEBUG nova.network.neutron [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Successfully created port: 9625f688-ed1b-4676-a575-789088c0a6e6 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 726.935557] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08eea116-d8eb-483f-9507-38f468bdef9d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 726.943629] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4bb2a3e-8096-43db-9861-fec8f378ec4a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 726.974992] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a0dfbb7-a88b-40ef-a3e7-62b3ec605b93 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 726.982332] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-720e5702-75c2-49d4-b981-96a5d7d9eae0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 726.986368] env[62499]: DEBUG nova.compute.manager [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 726.998575] env[62499]: DEBUG nova.compute.provider_tree [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 727.228736] env[62499]: DEBUG oslo_concurrency.lockutils [None req-051d8d39-24f6-407f-a99a-b7bf55e2f2f8 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "3d9ac347-ffcc-4840-8090-811dbd2865ef" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 123.546s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 727.501457] env[62499]: DEBUG nova.scheduler.client.report [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 727.731749] env[62499]: DEBUG nova.compute.manager [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 728.000139] env[62499]: DEBUG nova.compute.manager [req-88b721ba-d834-4f5a-a381-e0a0573b17c5 req-c7227b7a-2fb9-48d5-9b9b-f5dca5ce00f7 service nova] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Received event network-changed-9625f688-ed1b-4676-a575-789088c0a6e6 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 728.000348] env[62499]: DEBUG nova.compute.manager [req-88b721ba-d834-4f5a-a381-e0a0573b17c5 req-c7227b7a-2fb9-48d5-9b9b-f5dca5ce00f7 service nova] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Refreshing instance network info cache due to event network-changed-9625f688-ed1b-4676-a575-789088c0a6e6. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 728.000606] env[62499]: DEBUG oslo_concurrency.lockutils [req-88b721ba-d834-4f5a-a381-e0a0573b17c5 req-c7227b7a-2fb9-48d5-9b9b-f5dca5ce00f7 service nova] Acquiring lock "refresh_cache-8a769120-906c-4e7c-ac19-b7895039efc1" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 728.000752] env[62499]: DEBUG oslo_concurrency.lockutils [req-88b721ba-d834-4f5a-a381-e0a0573b17c5 req-c7227b7a-2fb9-48d5-9b9b-f5dca5ce00f7 service nova] Acquired lock "refresh_cache-8a769120-906c-4e7c-ac19-b7895039efc1" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 728.000913] env[62499]: DEBUG nova.network.neutron [req-88b721ba-d834-4f5a-a381-e0a0573b17c5 req-c7227b7a-2fb9-48d5-9b9b-f5dca5ce00f7 service nova] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Refreshing network info cache for port 9625f688-ed1b-4676-a575-789088c0a6e6 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 728.007176] env[62499]: DEBUG nova.compute.manager [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 728.013512] env[62499]: DEBUG oslo_concurrency.lockutils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.039s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 728.014177] env[62499]: ERROR nova.compute.manager [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 2d6c29d5-c728-42bd-86a1-1f0309315094, please check neutron logs for more information. [ 728.014177] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Traceback (most recent call last): [ 728.014177] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 728.014177] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] self.driver.spawn(context, instance, image_meta, [ 728.014177] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 728.014177] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 728.014177] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 728.014177] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] vm_ref = self.build_virtual_machine(instance, [ 728.014177] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 728.014177] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] vif_infos = vmwarevif.get_vif_info(self._session, [ 728.014177] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 728.014551] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] for vif in network_info: [ 728.014551] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 728.014551] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] return self._sync_wrapper(fn, *args, **kwargs) [ 728.014551] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 728.014551] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] self.wait() [ 728.014551] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 728.014551] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] self[:] = self._gt.wait() [ 728.014551] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 728.014551] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] return self._exit_event.wait() [ 728.014551] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 728.014551] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] current.throw(*self._exc) [ 728.014551] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 728.014551] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] result = function(*args, **kwargs) [ 728.014925] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 728.014925] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] return func(*args, **kwargs) [ 728.014925] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 728.014925] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] raise e [ 728.014925] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 728.014925] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] nwinfo = self.network_api.allocate_for_instance( [ 728.014925] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 728.014925] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] created_port_ids = self._update_ports_for_instance( [ 728.014925] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 728.014925] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] with excutils.save_and_reraise_exception(): [ 728.014925] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 728.014925] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] self.force_reraise() [ 728.014925] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 728.015320] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] raise self.value [ 728.015320] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 728.015320] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] updated_port = self._update_port( [ 728.015320] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 728.015320] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] _ensure_no_port_binding_failure(port) [ 728.015320] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 728.015320] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] raise exception.PortBindingFailed(port_id=port['id']) [ 728.015320] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] nova.exception.PortBindingFailed: Binding failed for port 2d6c29d5-c728-42bd-86a1-1f0309315094, please check neutron logs for more information. [ 728.015320] env[62499]: ERROR nova.compute.manager [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] [ 728.015320] env[62499]: DEBUG nova.compute.utils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Binding failed for port 2d6c29d5-c728-42bd-86a1-1f0309315094, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 728.016594] env[62499]: DEBUG oslo_concurrency.lockutils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.841s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 728.020136] env[62499]: DEBUG nova.compute.manager [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Build of instance 3649431e-f856-4a28-9af7-8f101c3fc2d6 was re-scheduled: Binding failed for port 2d6c29d5-c728-42bd-86a1-1f0309315094, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 728.020383] env[62499]: DEBUG nova.compute.manager [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 728.020625] env[62499]: DEBUG oslo_concurrency.lockutils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Acquiring lock "refresh_cache-3649431e-f856-4a28-9af7-8f101c3fc2d6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 728.020769] env[62499]: DEBUG oslo_concurrency.lockutils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Acquired lock "refresh_cache-3649431e-f856-4a28-9af7-8f101c3fc2d6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 728.021224] env[62499]: DEBUG nova.network.neutron [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 728.062670] env[62499]: DEBUG nova.virt.hardware [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 728.063189] env[62499]: DEBUG nova.virt.hardware [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 728.063189] env[62499]: DEBUG nova.virt.hardware [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 728.063311] env[62499]: DEBUG nova.virt.hardware [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 728.063426] env[62499]: DEBUG nova.virt.hardware [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 728.063824] env[62499]: DEBUG nova.virt.hardware [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 728.063824] env[62499]: DEBUG nova.virt.hardware [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 728.064020] env[62499]: DEBUG nova.virt.hardware [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 728.064196] env[62499]: DEBUG nova.virt.hardware [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 728.064369] env[62499]: DEBUG nova.virt.hardware [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 728.064547] env[62499]: DEBUG nova.virt.hardware [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 728.065704] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8719f51-43e7-4c6c-b7a9-11b80d772a05 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 728.074929] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2fff1a9-c1fe-4904-901d-424b01a808dc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 728.256808] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 728.325492] env[62499]: ERROR nova.compute.manager [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 9625f688-ed1b-4676-a575-789088c0a6e6, please check neutron logs for more information. [ 728.325492] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 728.325492] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 728.325492] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 728.325492] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 728.325492] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 728.325492] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 728.325492] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 728.325492] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 728.325492] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 728.325492] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 728.325492] env[62499]: ERROR nova.compute.manager raise self.value [ 728.325492] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 728.325492] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 728.325492] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 728.325492] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 728.326040] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 728.326040] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 728.326040] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 9625f688-ed1b-4676-a575-789088c0a6e6, please check neutron logs for more information. [ 728.326040] env[62499]: ERROR nova.compute.manager [ 728.326040] env[62499]: Traceback (most recent call last): [ 728.326040] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 728.326040] env[62499]: listener.cb(fileno) [ 728.326040] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 728.326040] env[62499]: result = function(*args, **kwargs) [ 728.326040] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 728.326040] env[62499]: return func(*args, **kwargs) [ 728.326040] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 728.326040] env[62499]: raise e [ 728.326040] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 728.326040] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 728.326040] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 728.326040] env[62499]: created_port_ids = self._update_ports_for_instance( [ 728.326040] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 728.326040] env[62499]: with excutils.save_and_reraise_exception(): [ 728.326040] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 728.326040] env[62499]: self.force_reraise() [ 728.326040] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 728.326040] env[62499]: raise self.value [ 728.326040] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 728.326040] env[62499]: updated_port = self._update_port( [ 728.326040] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 728.326040] env[62499]: _ensure_no_port_binding_failure(port) [ 728.326040] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 728.326040] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 728.327027] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 9625f688-ed1b-4676-a575-789088c0a6e6, please check neutron logs for more information. [ 728.327027] env[62499]: Removing descriptor: 17 [ 728.327027] env[62499]: ERROR nova.compute.manager [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 9625f688-ed1b-4676-a575-789088c0a6e6, please check neutron logs for more information. [ 728.327027] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Traceback (most recent call last): [ 728.327027] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 728.327027] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] yield resources [ 728.327027] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 728.327027] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] self.driver.spawn(context, instance, image_meta, [ 728.327027] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 728.327027] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 728.327027] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 728.327027] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] vm_ref = self.build_virtual_machine(instance, [ 728.327450] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 728.327450] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] vif_infos = vmwarevif.get_vif_info(self._session, [ 728.327450] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 728.327450] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] for vif in network_info: [ 728.327450] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 728.327450] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] return self._sync_wrapper(fn, *args, **kwargs) [ 728.327450] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 728.327450] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] self.wait() [ 728.327450] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 728.327450] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] self[:] = self._gt.wait() [ 728.327450] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 728.327450] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] return self._exit_event.wait() [ 728.327450] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 728.327863] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] result = hub.switch() [ 728.327863] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 728.327863] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] return self.greenlet.switch() [ 728.327863] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 728.327863] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] result = function(*args, **kwargs) [ 728.327863] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 728.327863] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] return func(*args, **kwargs) [ 728.327863] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 728.327863] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] raise e [ 728.327863] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 728.327863] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] nwinfo = self.network_api.allocate_for_instance( [ 728.327863] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 728.327863] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] created_port_ids = self._update_ports_for_instance( [ 728.328321] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 728.328321] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] with excutils.save_and_reraise_exception(): [ 728.328321] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 728.328321] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] self.force_reraise() [ 728.328321] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 728.328321] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] raise self.value [ 728.328321] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 728.328321] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] updated_port = self._update_port( [ 728.328321] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 728.328321] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] _ensure_no_port_binding_failure(port) [ 728.328321] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 728.328321] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] raise exception.PortBindingFailed(port_id=port['id']) [ 728.328674] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] nova.exception.PortBindingFailed: Binding failed for port 9625f688-ed1b-4676-a575-789088c0a6e6, please check neutron logs for more information. [ 728.328674] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] [ 728.328674] env[62499]: INFO nova.compute.manager [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Terminating instance [ 728.328770] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Acquiring lock "refresh_cache-8a769120-906c-4e7c-ac19-b7895039efc1" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 728.537072] env[62499]: DEBUG nova.network.neutron [req-88b721ba-d834-4f5a-a381-e0a0573b17c5 req-c7227b7a-2fb9-48d5-9b9b-f5dca5ce00f7 service nova] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 728.542015] env[62499]: DEBUG nova.network.neutron [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 728.645925] env[62499]: DEBUG nova.network.neutron [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 728.667555] env[62499]: DEBUG nova.network.neutron [req-88b721ba-d834-4f5a-a381-e0a0573b17c5 req-c7227b7a-2fb9-48d5-9b9b-f5dca5ce00f7 service nova] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 728.920320] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3c3113f-89c7-4326-baa0-255dfbd79d4e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 728.928806] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c0563253-0a61-4911-b574-176539f48af9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 728.959211] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-edb46289-5a3a-4780-a0c1-df1474fd7c94 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 728.966958] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ed53283-c01b-4123-8114-f9f9e8feef7e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 728.982025] env[62499]: DEBUG nova.compute.provider_tree [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 729.149593] env[62499]: DEBUG oslo_concurrency.lockutils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Releasing lock "refresh_cache-3649431e-f856-4a28-9af7-8f101c3fc2d6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 729.149593] env[62499]: DEBUG nova.compute.manager [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 729.149833] env[62499]: DEBUG nova.compute.manager [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 729.149875] env[62499]: DEBUG nova.network.neutron [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 729.164995] env[62499]: DEBUG nova.network.neutron [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 729.169049] env[62499]: DEBUG oslo_concurrency.lockutils [req-88b721ba-d834-4f5a-a381-e0a0573b17c5 req-c7227b7a-2fb9-48d5-9b9b-f5dca5ce00f7 service nova] Releasing lock "refresh_cache-8a769120-906c-4e7c-ac19-b7895039efc1" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 729.169508] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Acquired lock "refresh_cache-8a769120-906c-4e7c-ac19-b7895039efc1" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 729.169736] env[62499]: DEBUG nova.network.neutron [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 729.485192] env[62499]: DEBUG nova.scheduler.client.report [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 729.573756] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquiring lock "6da95995-0527-4b05-94dc-a4fcf0ae2f92" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 729.573992] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "6da95995-0527-4b05-94dc-a4fcf0ae2f92" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 729.668505] env[62499]: DEBUG nova.network.neutron [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 729.687093] env[62499]: DEBUG nova.network.neutron [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 729.767830] env[62499]: DEBUG nova.network.neutron [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 729.997434] env[62499]: DEBUG oslo_concurrency.lockutils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.981s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 729.998104] env[62499]: ERROR nova.compute.manager [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b10bfa9a-fa7b-425e-b992-d5db1489cc77, please check neutron logs for more information. [ 729.998104] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Traceback (most recent call last): [ 729.998104] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 729.998104] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] self.driver.spawn(context, instance, image_meta, [ 729.998104] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 729.998104] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] self._vmops.spawn(context, instance, image_meta, injected_files, [ 729.998104] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 729.998104] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] vm_ref = self.build_virtual_machine(instance, [ 729.998104] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 729.998104] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] vif_infos = vmwarevif.get_vif_info(self._session, [ 729.998104] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 729.998497] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] for vif in network_info: [ 729.998497] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 729.998497] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] return self._sync_wrapper(fn, *args, **kwargs) [ 729.998497] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 729.998497] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] self.wait() [ 729.998497] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 729.998497] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] self[:] = self._gt.wait() [ 729.998497] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 729.998497] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] return self._exit_event.wait() [ 729.998497] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 729.998497] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] current.throw(*self._exc) [ 729.998497] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 729.998497] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] result = function(*args, **kwargs) [ 729.998919] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 729.998919] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] return func(*args, **kwargs) [ 729.998919] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 729.998919] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] raise e [ 729.998919] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 729.998919] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] nwinfo = self.network_api.allocate_for_instance( [ 729.998919] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 729.998919] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] created_port_ids = self._update_ports_for_instance( [ 729.998919] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 729.998919] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] with excutils.save_and_reraise_exception(): [ 729.998919] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 729.998919] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] self.force_reraise() [ 729.998919] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 729.999344] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] raise self.value [ 729.999344] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 729.999344] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] updated_port = self._update_port( [ 729.999344] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 729.999344] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] _ensure_no_port_binding_failure(port) [ 729.999344] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 729.999344] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] raise exception.PortBindingFailed(port_id=port['id']) [ 729.999344] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] nova.exception.PortBindingFailed: Binding failed for port b10bfa9a-fa7b-425e-b992-d5db1489cc77, please check neutron logs for more information. [ 729.999344] env[62499]: ERROR nova.compute.manager [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] [ 729.999344] env[62499]: DEBUG nova.compute.utils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Binding failed for port b10bfa9a-fa7b-425e-b992-d5db1489cc77, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 730.000077] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.345s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 730.001527] env[62499]: INFO nova.compute.claims [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 730.004077] env[62499]: DEBUG nova.compute.manager [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Build of instance 71fa08bc-a5c7-459c-b65b-5dacd9537281 was re-scheduled: Binding failed for port b10bfa9a-fa7b-425e-b992-d5db1489cc77, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 730.004499] env[62499]: DEBUG nova.compute.manager [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 730.004722] env[62499]: DEBUG oslo_concurrency.lockutils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "refresh_cache-71fa08bc-a5c7-459c-b65b-5dacd9537281" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 730.004866] env[62499]: DEBUG oslo_concurrency.lockutils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquired lock "refresh_cache-71fa08bc-a5c7-459c-b65b-5dacd9537281" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 730.005033] env[62499]: DEBUG nova.network.neutron [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 730.070193] env[62499]: DEBUG nova.compute.manager [req-82798f25-8d43-4e6b-8dd1-add45081149c req-c8e811a8-ee8a-44f3-ac5b-4d5611789038 service nova] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Received event network-vif-deleted-9625f688-ed1b-4676-a575-789088c0a6e6 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 730.171339] env[62499]: INFO nova.compute.manager [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] [instance: 3649431e-f856-4a28-9af7-8f101c3fc2d6] Took 1.02 seconds to deallocate network for instance. [ 730.273024] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Releasing lock "refresh_cache-8a769120-906c-4e7c-ac19-b7895039efc1" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 730.273024] env[62499]: DEBUG nova.compute.manager [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 730.273024] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 730.273024] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-416ccefa-00fd-4637-b19a-84c2b2c74841 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 730.281899] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca79ea09-ab4b-40a4-807b-dda0b8646e94 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 730.305438] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 8a769120-906c-4e7c-ac19-b7895039efc1 could not be found. [ 730.305737] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 730.305975] env[62499]: INFO nova.compute.manager [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Took 0.04 seconds to destroy the instance on the hypervisor. [ 730.306303] env[62499]: DEBUG oslo.service.loopingcall [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 730.306635] env[62499]: DEBUG nova.compute.manager [-] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 730.306785] env[62499]: DEBUG nova.network.neutron [-] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 730.328332] env[62499]: DEBUG nova.network.neutron [-] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 730.530766] env[62499]: DEBUG nova.network.neutron [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 730.644188] env[62499]: DEBUG nova.network.neutron [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 730.835922] env[62499]: DEBUG nova.network.neutron [-] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 731.147556] env[62499]: DEBUG oslo_concurrency.lockutils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Releasing lock "refresh_cache-71fa08bc-a5c7-459c-b65b-5dacd9537281" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 731.147916] env[62499]: DEBUG nova.compute.manager [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 731.147975] env[62499]: DEBUG nova.compute.manager [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 731.148156] env[62499]: DEBUG nova.network.neutron [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 731.164145] env[62499]: DEBUG nova.network.neutron [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 731.199845] env[62499]: INFO nova.scheduler.client.report [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Deleted allocations for instance 3649431e-f856-4a28-9af7-8f101c3fc2d6 [ 731.340440] env[62499]: INFO nova.compute.manager [-] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Took 1.03 seconds to deallocate network for instance. [ 731.342930] env[62499]: DEBUG nova.compute.claims [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 731.343874] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 731.350923] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8edd64cd-cdb2-4501-ba58-50e13f7985f4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 731.358948] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5e3017b-ebe7-4d28-8a0e-ed6b236c900e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 731.388060] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a02e15a8-cfd7-44f5-94b2-e7169cbce365 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 731.395764] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-924cecd7-714f-4609-8b1b-6db91faf4c36 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 731.408738] env[62499]: DEBUG nova.compute.provider_tree [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 731.667707] env[62499]: DEBUG nova.network.neutron [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 731.711049] env[62499]: DEBUG oslo_concurrency.lockutils [None req-85f236cd-415a-4fa9-8043-0de46fbbe27c tempest-ServersTestManualDisk-920724852 tempest-ServersTestManualDisk-920724852-project-member] Lock "3649431e-f856-4a28-9af7-8f101c3fc2d6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 125.041s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 731.912340] env[62499]: DEBUG nova.scheduler.client.report [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 732.170750] env[62499]: INFO nova.compute.manager [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 71fa08bc-a5c7-459c-b65b-5dacd9537281] Took 1.02 seconds to deallocate network for instance. [ 732.213866] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 732.420020] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.417s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 732.420020] env[62499]: DEBUG nova.compute.manager [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 732.420956] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.345s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 732.733161] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 732.925581] env[62499]: DEBUG nova.compute.utils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 732.930039] env[62499]: DEBUG nova.compute.manager [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 732.930039] env[62499]: DEBUG nova.network.neutron [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 733.159179] env[62499]: DEBUG nova.policy [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c92b9d37f796479ca3b6d9f0ed1789c8', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e444950a637b4fdab2edb1d62aac4be5', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 733.201692] env[62499]: INFO nova.scheduler.client.report [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Deleted allocations for instance 71fa08bc-a5c7-459c-b65b-5dacd9537281 [ 733.379470] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5dab332-df03-498d-afd7-8c4ba6052882 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 733.389521] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00b9781a-1b2f-40a6-ad95-f9a3fc6270d8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 733.420260] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7439dae9-be18-4125-8cfe-74101b88d933 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 733.430102] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe642164-1698-4c24-b22d-f914bb6ed120 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 733.434387] env[62499]: DEBUG nova.compute.manager [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 733.451631] env[62499]: DEBUG nova.compute.provider_tree [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 733.482376] env[62499]: DEBUG nova.network.neutron [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Successfully created port: b5066e31-5b07-4469-91f7-8ef4b5ef88da {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 733.714389] env[62499]: DEBUG oslo_concurrency.lockutils [None req-462c970d-8e5c-46b5-a82b-3be7d8a26e3a tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "71fa08bc-a5c7-459c-b65b-5dacd9537281" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 124.507s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 733.953281] env[62499]: DEBUG nova.scheduler.client.report [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 734.215411] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 734.451691] env[62499]: DEBUG nova.compute.manager [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 734.461533] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.040s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 734.462280] env[62499]: ERROR nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 89eccc4e-8949-458d-a72c-cb35a3c3f973, please check neutron logs for more information. [ 734.462280] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Traceback (most recent call last): [ 734.462280] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 734.462280] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] self.driver.spawn(context, instance, image_meta, [ 734.462280] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 734.462280] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 734.462280] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 734.462280] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] vm_ref = self.build_virtual_machine(instance, [ 734.462280] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 734.462280] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] vif_infos = vmwarevif.get_vif_info(self._session, [ 734.462280] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 734.462716] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] for vif in network_info: [ 734.462716] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 734.462716] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] return self._sync_wrapper(fn, *args, **kwargs) [ 734.462716] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 734.462716] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] self.wait() [ 734.462716] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 734.462716] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] self[:] = self._gt.wait() [ 734.462716] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 734.462716] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] return self._exit_event.wait() [ 734.462716] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 734.462716] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] current.throw(*self._exc) [ 734.462716] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 734.462716] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] result = function(*args, **kwargs) [ 734.463144] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 734.463144] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] return func(*args, **kwargs) [ 734.463144] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 734.463144] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] raise e [ 734.463144] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 734.463144] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] nwinfo = self.network_api.allocate_for_instance( [ 734.463144] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 734.463144] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] created_port_ids = self._update_ports_for_instance( [ 734.463144] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 734.463144] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] with excutils.save_and_reraise_exception(): [ 734.463144] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 734.463144] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] self.force_reraise() [ 734.463144] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 734.463638] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] raise self.value [ 734.463638] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 734.463638] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] updated_port = self._update_port( [ 734.463638] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 734.463638] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] _ensure_no_port_binding_failure(port) [ 734.463638] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 734.463638] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] raise exception.PortBindingFailed(port_id=port['id']) [ 734.463638] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] nova.exception.PortBindingFailed: Binding failed for port 89eccc4e-8949-458d-a72c-cb35a3c3f973, please check neutron logs for more information. [ 734.463638] env[62499]: ERROR nova.compute.manager [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] [ 734.463638] env[62499]: DEBUG nova.compute.utils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Binding failed for port 89eccc4e-8949-458d-a72c-cb35a3c3f973, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 734.465232] env[62499]: DEBUG nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Build of instance 6ae11b59-68af-482a-ad65-543a56d7aaa4 was re-scheduled: Binding failed for port 89eccc4e-8949-458d-a72c-cb35a3c3f973, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 734.465659] env[62499]: DEBUG nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 734.467648] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Acquiring lock "refresh_cache-6ae11b59-68af-482a-ad65-543a56d7aaa4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 734.467934] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Acquired lock "refresh_cache-6ae11b59-68af-482a-ad65-543a56d7aaa4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 734.468530] env[62499]: DEBUG nova.network.neutron [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 734.469541] env[62499]: DEBUG oslo_concurrency.lockutils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.137s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 734.471645] env[62499]: INFO nova.compute.claims [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 734.496705] env[62499]: DEBUG nova.virt.hardware [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 734.496960] env[62499]: DEBUG nova.virt.hardware [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 734.497142] env[62499]: DEBUG nova.virt.hardware [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 734.497359] env[62499]: DEBUG nova.virt.hardware [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 734.497539] env[62499]: DEBUG nova.virt.hardware [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 734.497847] env[62499]: DEBUG nova.virt.hardware [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 734.499579] env[62499]: DEBUG nova.virt.hardware [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 734.499579] env[62499]: DEBUG nova.virt.hardware [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 734.499579] env[62499]: DEBUG nova.virt.hardware [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 734.499579] env[62499]: DEBUG nova.virt.hardware [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 734.499579] env[62499]: DEBUG nova.virt.hardware [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 734.500376] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae75bacb-9d3c-4136-af42-e8a1d2030a1a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 734.513332] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cbc2f23e-b332-4035-b9b6-64c7e4e57bae {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 734.743470] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 734.828334] env[62499]: DEBUG nova.compute.manager [req-8c6631a7-2bf1-46c6-bd2a-d26d6e61021d req-00ee90af-851c-4525-9994-ecfb72f8925e service nova] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Received event network-changed-b5066e31-5b07-4469-91f7-8ef4b5ef88da {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 734.828535] env[62499]: DEBUG nova.compute.manager [req-8c6631a7-2bf1-46c6-bd2a-d26d6e61021d req-00ee90af-851c-4525-9994-ecfb72f8925e service nova] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Refreshing instance network info cache due to event network-changed-b5066e31-5b07-4469-91f7-8ef4b5ef88da. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 734.828814] env[62499]: DEBUG oslo_concurrency.lockutils [req-8c6631a7-2bf1-46c6-bd2a-d26d6e61021d req-00ee90af-851c-4525-9994-ecfb72f8925e service nova] Acquiring lock "refresh_cache-75b04055-8c7e-4659-9451-ddcd6d39fe2e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 734.828987] env[62499]: DEBUG oslo_concurrency.lockutils [req-8c6631a7-2bf1-46c6-bd2a-d26d6e61021d req-00ee90af-851c-4525-9994-ecfb72f8925e service nova] Acquired lock "refresh_cache-75b04055-8c7e-4659-9451-ddcd6d39fe2e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 734.830030] env[62499]: DEBUG nova.network.neutron [req-8c6631a7-2bf1-46c6-bd2a-d26d6e61021d req-00ee90af-851c-4525-9994-ecfb72f8925e service nova] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Refreshing network info cache for port b5066e31-5b07-4469-91f7-8ef4b5ef88da {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 735.004831] env[62499]: ERROR nova.compute.manager [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b5066e31-5b07-4469-91f7-8ef4b5ef88da, please check neutron logs for more information. [ 735.004831] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 735.004831] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 735.004831] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 735.004831] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 735.004831] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 735.004831] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 735.004831] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 735.004831] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 735.004831] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 735.004831] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 735.004831] env[62499]: ERROR nova.compute.manager raise self.value [ 735.004831] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 735.004831] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 735.004831] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 735.004831] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 735.005411] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 735.005411] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 735.005411] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b5066e31-5b07-4469-91f7-8ef4b5ef88da, please check neutron logs for more information. [ 735.005411] env[62499]: ERROR nova.compute.manager [ 735.005411] env[62499]: Traceback (most recent call last): [ 735.005411] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 735.005411] env[62499]: listener.cb(fileno) [ 735.005411] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 735.005411] env[62499]: result = function(*args, **kwargs) [ 735.005411] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 735.005411] env[62499]: return func(*args, **kwargs) [ 735.005411] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 735.005411] env[62499]: raise e [ 735.005411] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 735.005411] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 735.005411] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 735.005411] env[62499]: created_port_ids = self._update_ports_for_instance( [ 735.005411] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 735.005411] env[62499]: with excutils.save_and_reraise_exception(): [ 735.005411] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 735.005411] env[62499]: self.force_reraise() [ 735.005411] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 735.005411] env[62499]: raise self.value [ 735.005411] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 735.005411] env[62499]: updated_port = self._update_port( [ 735.005411] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 735.005411] env[62499]: _ensure_no_port_binding_failure(port) [ 735.005411] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 735.005411] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 735.006297] env[62499]: nova.exception.PortBindingFailed: Binding failed for port b5066e31-5b07-4469-91f7-8ef4b5ef88da, please check neutron logs for more information. [ 735.006297] env[62499]: Removing descriptor: 17 [ 735.006297] env[62499]: ERROR nova.compute.manager [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b5066e31-5b07-4469-91f7-8ef4b5ef88da, please check neutron logs for more information. [ 735.006297] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Traceback (most recent call last): [ 735.006297] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 735.006297] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] yield resources [ 735.006297] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 735.006297] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] self.driver.spawn(context, instance, image_meta, [ 735.006297] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 735.006297] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 735.006297] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 735.006297] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] vm_ref = self.build_virtual_machine(instance, [ 735.006688] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 735.006688] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] vif_infos = vmwarevif.get_vif_info(self._session, [ 735.006688] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 735.006688] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] for vif in network_info: [ 735.006688] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 735.006688] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] return self._sync_wrapper(fn, *args, **kwargs) [ 735.006688] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 735.006688] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] self.wait() [ 735.006688] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 735.006688] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] self[:] = self._gt.wait() [ 735.006688] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 735.006688] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] return self._exit_event.wait() [ 735.006688] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 735.007089] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] result = hub.switch() [ 735.007089] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 735.007089] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] return self.greenlet.switch() [ 735.007089] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 735.007089] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] result = function(*args, **kwargs) [ 735.007089] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 735.007089] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] return func(*args, **kwargs) [ 735.007089] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 735.007089] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] raise e [ 735.007089] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 735.007089] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] nwinfo = self.network_api.allocate_for_instance( [ 735.007089] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 735.007089] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] created_port_ids = self._update_ports_for_instance( [ 735.007531] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 735.007531] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] with excutils.save_and_reraise_exception(): [ 735.007531] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 735.007531] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] self.force_reraise() [ 735.007531] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 735.007531] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] raise self.value [ 735.007531] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 735.007531] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] updated_port = self._update_port( [ 735.007531] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 735.007531] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] _ensure_no_port_binding_failure(port) [ 735.007531] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 735.007531] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] raise exception.PortBindingFailed(port_id=port['id']) [ 735.007909] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] nova.exception.PortBindingFailed: Binding failed for port b5066e31-5b07-4469-91f7-8ef4b5ef88da, please check neutron logs for more information. [ 735.007909] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] [ 735.007909] env[62499]: INFO nova.compute.manager [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Terminating instance [ 735.009637] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "refresh_cache-75b04055-8c7e-4659-9451-ddcd6d39fe2e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 735.013052] env[62499]: DEBUG nova.network.neutron [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 735.097074] env[62499]: DEBUG nova.network.neutron [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 735.347016] env[62499]: DEBUG nova.network.neutron [req-8c6631a7-2bf1-46c6-bd2a-d26d6e61021d req-00ee90af-851c-4525-9994-ecfb72f8925e service nova] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 735.440045] env[62499]: DEBUG nova.network.neutron [req-8c6631a7-2bf1-46c6-bd2a-d26d6e61021d req-00ee90af-851c-4525-9994-ecfb72f8925e service nova] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 735.440045] env[62499]: DEBUG oslo_concurrency.lockutils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "263931cd-b2dc-41bb-8a2b-abf61aadafc9" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.440045] env[62499]: DEBUG oslo_concurrency.lockutils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "263931cd-b2dc-41bb-8a2b-abf61aadafc9" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.603664] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Releasing lock "refresh_cache-6ae11b59-68af-482a-ad65-543a56d7aaa4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 735.603962] env[62499]: DEBUG nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 735.604172] env[62499]: DEBUG nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 735.604344] env[62499]: DEBUG nova.network.neutron [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 735.619092] env[62499]: DEBUG nova.network.neutron [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 735.829663] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4207af4f-2dec-4aa5-b171-3af40378d7cd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 735.837748] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c72ad6d5-db70-43d4-9389-60cd732cf94f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 735.866473] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2509ef4a-3393-406d-a0a8-b7e99aa89af3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 735.874418] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b74120e9-9d39-4ea7-bd76-9eccb779af69 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 735.888176] env[62499]: DEBUG nova.compute.provider_tree [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 735.942148] env[62499]: DEBUG oslo_concurrency.lockutils [req-8c6631a7-2bf1-46c6-bd2a-d26d6e61021d req-00ee90af-851c-4525-9994-ecfb72f8925e service nova] Releasing lock "refresh_cache-75b04055-8c7e-4659-9451-ddcd6d39fe2e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 735.942542] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquired lock "refresh_cache-75b04055-8c7e-4659-9451-ddcd6d39fe2e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 735.942727] env[62499]: DEBUG nova.network.neutron [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 736.124655] env[62499]: DEBUG nova.network.neutron [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 736.391498] env[62499]: DEBUG nova.scheduler.client.report [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 736.477070] env[62499]: DEBUG nova.network.neutron [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 736.614566] env[62499]: DEBUG nova.network.neutron [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 736.627650] env[62499]: INFO nova.compute.manager [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] [instance: 6ae11b59-68af-482a-ad65-543a56d7aaa4] Took 1.02 seconds to deallocate network for instance. [ 736.864565] env[62499]: DEBUG nova.compute.manager [req-dc102d4b-9eb9-4646-806a-6cbac916031a req-60cff13d-8111-4c1f-a646-a6b503e911cd service nova] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Received event network-vif-deleted-b5066e31-5b07-4469-91f7-8ef4b5ef88da {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 736.897012] env[62499]: DEBUG oslo_concurrency.lockutils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.427s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 736.897846] env[62499]: DEBUG nova.compute.manager [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 736.904028] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.482s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 737.117648] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Releasing lock "refresh_cache-75b04055-8c7e-4659-9451-ddcd6d39fe2e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 737.118083] env[62499]: DEBUG nova.compute.manager [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 737.118275] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 737.118573] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ab2af0c6-7c71-42c8-a585-f12a19ad1102 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 737.130713] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5717ef1e-caf3-40c0-a05f-326df029d74c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 737.160131] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 75b04055-8c7e-4659-9451-ddcd6d39fe2e could not be found. [ 737.160131] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 737.160131] env[62499]: INFO nova.compute.manager [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Took 0.04 seconds to destroy the instance on the hypervisor. [ 737.160362] env[62499]: DEBUG oslo.service.loopingcall [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 737.161508] env[62499]: DEBUG nova.compute.manager [-] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 737.161508] env[62499]: DEBUG nova.network.neutron [-] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 737.186777] env[62499]: DEBUG nova.network.neutron [-] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 737.410261] env[62499]: DEBUG nova.compute.utils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 737.411737] env[62499]: DEBUG nova.compute.manager [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 737.411997] env[62499]: DEBUG nova.network.neutron [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 737.482904] env[62499]: DEBUG nova.policy [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4b2a770097844555942cf8293c401aaf', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '74849557e5494e259bf7824762dc75a3', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 737.676844] env[62499]: INFO nova.scheduler.client.report [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Deleted allocations for instance 6ae11b59-68af-482a-ad65-543a56d7aaa4 [ 737.688773] env[62499]: DEBUG nova.network.neutron [-] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 737.881282] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e4de315-74dc-488d-8859-2f71dc2975af {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 737.890539] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-13a49195-8643-4b25-ad6d-8a58ef671970 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 737.923023] env[62499]: DEBUG nova.compute.manager [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 737.927127] env[62499]: DEBUG nova.network.neutron [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Successfully created port: ab3d96c4-7e66-4d1c-9ee7-2acc85624f4a {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 737.930183] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-25738158-2dc3-4892-9e6e-1696032ee1a3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 737.937458] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4767bc12-754b-4d94-9f06-75bb8f6822c4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 737.951124] env[62499]: DEBUG nova.compute.provider_tree [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 738.188337] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b936d681-794c-4669-a040-bf8c73dee3ec tempest-ServerActionsV293TestJSON-154139406 tempest-ServerActionsV293TestJSON-154139406-project-member] Lock "6ae11b59-68af-482a-ad65-543a56d7aaa4" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 128.500s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 738.192290] env[62499]: INFO nova.compute.manager [-] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Took 1.03 seconds to deallocate network for instance. [ 738.194034] env[62499]: DEBUG nova.compute.claims [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 738.194299] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 738.458260] env[62499]: DEBUG nova.scheduler.client.report [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 738.692024] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 738.933369] env[62499]: DEBUG nova.compute.manager [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 738.962054] env[62499]: DEBUG nova.virt.hardware [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 738.962276] env[62499]: DEBUG nova.virt.hardware [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 738.962507] env[62499]: DEBUG nova.virt.hardware [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 738.962591] env[62499]: DEBUG nova.virt.hardware [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 738.962735] env[62499]: DEBUG nova.virt.hardware [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 738.962879] env[62499]: DEBUG nova.virt.hardware [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 738.963113] env[62499]: DEBUG nova.virt.hardware [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 738.963320] env[62499]: DEBUG nova.virt.hardware [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 738.963406] env[62499]: DEBUG nova.virt.hardware [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 738.963947] env[62499]: DEBUG nova.virt.hardware [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 738.963947] env[62499]: DEBUG nova.virt.hardware [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 738.964437] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.063s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 738.964998] env[62499]: ERROR nova.compute.manager [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d4db130b-8be3-4540-8b5d-cec6c5fc0da5, please check neutron logs for more information. [ 738.964998] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Traceback (most recent call last): [ 738.964998] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 738.964998] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] self.driver.spawn(context, instance, image_meta, [ 738.964998] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 738.964998] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] self._vmops.spawn(context, instance, image_meta, injected_files, [ 738.964998] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 738.964998] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] vm_ref = self.build_virtual_machine(instance, [ 738.964998] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 738.964998] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] vif_infos = vmwarevif.get_vif_info(self._session, [ 738.964998] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 738.965438] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] for vif in network_info: [ 738.965438] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 738.965438] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] return self._sync_wrapper(fn, *args, **kwargs) [ 738.965438] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 738.965438] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] self.wait() [ 738.965438] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 738.965438] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] self[:] = self._gt.wait() [ 738.965438] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 738.965438] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] return self._exit_event.wait() [ 738.965438] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 738.965438] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] result = hub.switch() [ 738.965438] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 738.965438] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] return self.greenlet.switch() [ 738.965864] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 738.965864] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] result = function(*args, **kwargs) [ 738.965864] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 738.965864] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] return func(*args, **kwargs) [ 738.965864] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 738.965864] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] raise e [ 738.965864] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 738.965864] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] nwinfo = self.network_api.allocate_for_instance( [ 738.965864] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 738.965864] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] created_port_ids = self._update_ports_for_instance( [ 738.965864] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 738.965864] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] with excutils.save_and_reraise_exception(): [ 738.965864] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 738.966308] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] self.force_reraise() [ 738.966308] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 738.966308] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] raise self.value [ 738.966308] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 738.966308] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] updated_port = self._update_port( [ 738.966308] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 738.966308] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] _ensure_no_port_binding_failure(port) [ 738.966308] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 738.966308] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] raise exception.PortBindingFailed(port_id=port['id']) [ 738.966308] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] nova.exception.PortBindingFailed: Binding failed for port d4db130b-8be3-4540-8b5d-cec6c5fc0da5, please check neutron logs for more information. [ 738.966308] env[62499]: ERROR nova.compute.manager [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] [ 738.967367] env[62499]: DEBUG nova.compute.utils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Binding failed for port d4db130b-8be3-4540-8b5d-cec6c5fc0da5, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 738.968187] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-623db2fc-adbd-4bcb-bf1f-b16d800e021d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 738.970797] env[62499]: DEBUG nova.compute.manager [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Build of instance 6d8b7855-567c-44c3-90ee-fc0daec87f46 was re-scheduled: Binding failed for port d4db130b-8be3-4540-8b5d-cec6c5fc0da5, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 738.971419] env[62499]: DEBUG nova.compute.manager [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 738.971504] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Acquiring lock "refresh_cache-6d8b7855-567c-44c3-90ee-fc0daec87f46" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 738.971760] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Acquired lock "refresh_cache-6d8b7855-567c-44c3-90ee-fc0daec87f46" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 738.971760] env[62499]: DEBUG nova.network.neutron [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 738.972854] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.967s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 738.974473] env[62499]: INFO nova.compute.claims [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 738.982774] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47bd3f5b-4980-4578-a795-e37a6d3d769a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 739.191767] env[62499]: DEBUG nova.compute.manager [req-e8304bf1-e0c4-4881-a273-070a41d3a2d0 req-e71e132b-c149-4a05-9f6a-f535a6336a52 service nova] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Received event network-changed-ab3d96c4-7e66-4d1c-9ee7-2acc85624f4a {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 739.192272] env[62499]: DEBUG nova.compute.manager [req-e8304bf1-e0c4-4881-a273-070a41d3a2d0 req-e71e132b-c149-4a05-9f6a-f535a6336a52 service nova] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Refreshing instance network info cache due to event network-changed-ab3d96c4-7e66-4d1c-9ee7-2acc85624f4a. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 739.192272] env[62499]: DEBUG oslo_concurrency.lockutils [req-e8304bf1-e0c4-4881-a273-070a41d3a2d0 req-e71e132b-c149-4a05-9f6a-f535a6336a52 service nova] Acquiring lock "refresh_cache-78a83324-36da-457e-a78d-c82d0a722015" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 739.192400] env[62499]: DEBUG oslo_concurrency.lockutils [req-e8304bf1-e0c4-4881-a273-070a41d3a2d0 req-e71e132b-c149-4a05-9f6a-f535a6336a52 service nova] Acquired lock "refresh_cache-78a83324-36da-457e-a78d-c82d0a722015" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 739.192602] env[62499]: DEBUG nova.network.neutron [req-e8304bf1-e0c4-4881-a273-070a41d3a2d0 req-e71e132b-c149-4a05-9f6a-f535a6336a52 service nova] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Refreshing network info cache for port ab3d96c4-7e66-4d1c-9ee7-2acc85624f4a {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 739.224316] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 739.343283] env[62499]: ERROR nova.compute.manager [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ab3d96c4-7e66-4d1c-9ee7-2acc85624f4a, please check neutron logs for more information. [ 739.343283] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 739.343283] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 739.343283] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 739.343283] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 739.343283] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 739.343283] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 739.343283] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 739.343283] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 739.343283] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 739.343283] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 739.343283] env[62499]: ERROR nova.compute.manager raise self.value [ 739.343283] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 739.343283] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 739.343283] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 739.343283] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 739.344121] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 739.344121] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 739.344121] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ab3d96c4-7e66-4d1c-9ee7-2acc85624f4a, please check neutron logs for more information. [ 739.344121] env[62499]: ERROR nova.compute.manager [ 739.344121] env[62499]: Traceback (most recent call last): [ 739.344121] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 739.344121] env[62499]: listener.cb(fileno) [ 739.344121] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 739.344121] env[62499]: result = function(*args, **kwargs) [ 739.344121] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 739.344121] env[62499]: return func(*args, **kwargs) [ 739.344121] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 739.344121] env[62499]: raise e [ 739.344121] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 739.344121] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 739.344121] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 739.344121] env[62499]: created_port_ids = self._update_ports_for_instance( [ 739.344121] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 739.344121] env[62499]: with excutils.save_and_reraise_exception(): [ 739.344121] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 739.344121] env[62499]: self.force_reraise() [ 739.344121] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 739.344121] env[62499]: raise self.value [ 739.344121] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 739.344121] env[62499]: updated_port = self._update_port( [ 739.344121] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 739.344121] env[62499]: _ensure_no_port_binding_failure(port) [ 739.344121] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 739.344121] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 739.345626] env[62499]: nova.exception.PortBindingFailed: Binding failed for port ab3d96c4-7e66-4d1c-9ee7-2acc85624f4a, please check neutron logs for more information. [ 739.345626] env[62499]: Removing descriptor: 16 [ 739.345626] env[62499]: ERROR nova.compute.manager [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ab3d96c4-7e66-4d1c-9ee7-2acc85624f4a, please check neutron logs for more information. [ 739.345626] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] Traceback (most recent call last): [ 739.345626] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 739.345626] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] yield resources [ 739.345626] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 739.345626] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] self.driver.spawn(context, instance, image_meta, [ 739.345626] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 739.345626] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] self._vmops.spawn(context, instance, image_meta, injected_files, [ 739.345626] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 739.345626] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] vm_ref = self.build_virtual_machine(instance, [ 739.346248] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 739.346248] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] vif_infos = vmwarevif.get_vif_info(self._session, [ 739.346248] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 739.346248] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] for vif in network_info: [ 739.346248] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 739.346248] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] return self._sync_wrapper(fn, *args, **kwargs) [ 739.346248] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 739.346248] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] self.wait() [ 739.346248] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 739.346248] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] self[:] = self._gt.wait() [ 739.346248] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 739.346248] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] return self._exit_event.wait() [ 739.346248] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 739.346880] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] result = hub.switch() [ 739.346880] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 739.346880] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] return self.greenlet.switch() [ 739.346880] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 739.346880] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] result = function(*args, **kwargs) [ 739.346880] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 739.346880] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] return func(*args, **kwargs) [ 739.346880] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 739.346880] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] raise e [ 739.346880] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 739.346880] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] nwinfo = self.network_api.allocate_for_instance( [ 739.346880] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 739.346880] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] created_port_ids = self._update_ports_for_instance( [ 739.347622] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 739.347622] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] with excutils.save_and_reraise_exception(): [ 739.347622] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 739.347622] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] self.force_reraise() [ 739.347622] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 739.347622] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] raise self.value [ 739.347622] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 739.347622] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] updated_port = self._update_port( [ 739.347622] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 739.347622] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] _ensure_no_port_binding_failure(port) [ 739.347622] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 739.347622] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] raise exception.PortBindingFailed(port_id=port['id']) [ 739.348176] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] nova.exception.PortBindingFailed: Binding failed for port ab3d96c4-7e66-4d1c-9ee7-2acc85624f4a, please check neutron logs for more information. [ 739.348176] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] [ 739.348176] env[62499]: INFO nova.compute.manager [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Terminating instance [ 739.348176] env[62499]: DEBUG oslo_concurrency.lockutils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Acquiring lock "refresh_cache-78a83324-36da-457e-a78d-c82d0a722015" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 739.508157] env[62499]: DEBUG nova.network.neutron [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 739.625040] env[62499]: DEBUG nova.network.neutron [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 739.711509] env[62499]: DEBUG nova.network.neutron [req-e8304bf1-e0c4-4881-a273-070a41d3a2d0 req-e71e132b-c149-4a05-9f6a-f535a6336a52 service nova] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 739.791224] env[62499]: DEBUG nova.network.neutron [req-e8304bf1-e0c4-4881-a273-070a41d3a2d0 req-e71e132b-c149-4a05-9f6a-f535a6336a52 service nova] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 740.128130] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Releasing lock "refresh_cache-6d8b7855-567c-44c3-90ee-fc0daec87f46" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 740.129028] env[62499]: DEBUG nova.compute.manager [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 740.129028] env[62499]: DEBUG nova.compute.manager [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 740.129028] env[62499]: DEBUG nova.network.neutron [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 740.160873] env[62499]: DEBUG nova.network.neutron [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 740.292934] env[62499]: DEBUG oslo_concurrency.lockutils [req-e8304bf1-e0c4-4881-a273-070a41d3a2d0 req-e71e132b-c149-4a05-9f6a-f535a6336a52 service nova] Releasing lock "refresh_cache-78a83324-36da-457e-a78d-c82d0a722015" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 740.293813] env[62499]: DEBUG oslo_concurrency.lockutils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Acquired lock "refresh_cache-78a83324-36da-457e-a78d-c82d0a722015" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 740.295166] env[62499]: DEBUG nova.network.neutron [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 740.562638] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14723bd7-dc53-4378-8e11-c69134d1e284 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.571249] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ea2dd2e-bc79-4449-a6c6-2c56016bd5da {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.610556] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee5200eb-9b8a-43d6-9bad-233532e60a5a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.623293] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d297e8fa-b165-4b28-9951-7c2ef2dfd8fe {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.633741] env[62499]: DEBUG nova.compute.provider_tree [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 740.665692] env[62499]: DEBUG nova.network.neutron [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 740.838330] env[62499]: DEBUG nova.network.neutron [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 741.061834] env[62499]: DEBUG nova.network.neutron [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 741.143196] env[62499]: DEBUG nova.scheduler.client.report [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 741.168334] env[62499]: INFO nova.compute.manager [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] [instance: 6d8b7855-567c-44c3-90ee-fc0daec87f46] Took 1.04 seconds to deallocate network for instance. [ 741.239926] env[62499]: DEBUG nova.compute.manager [req-4ed65637-4fde-4594-a53c-133da4461666 req-35255031-4100-4276-b333-47a446b522db service nova] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Received event network-vif-deleted-ab3d96c4-7e66-4d1c-9ee7-2acc85624f4a {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 741.569230] env[62499]: DEBUG oslo_concurrency.lockutils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Releasing lock "refresh_cache-78a83324-36da-457e-a78d-c82d0a722015" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 741.569525] env[62499]: DEBUG nova.compute.manager [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 741.569724] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 741.570345] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-42719a26-1d5e-4825-b826-c53402b4f588 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 741.584031] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b825ddd2-ffee-4ed1-bf4a-ec5709560a9a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 741.610095] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 78a83324-36da-457e-a78d-c82d0a722015 could not be found. [ 741.610532] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 741.610612] env[62499]: INFO nova.compute.manager [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Took 0.04 seconds to destroy the instance on the hypervisor. [ 741.610820] env[62499]: DEBUG oslo.service.loopingcall [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 741.611107] env[62499]: DEBUG nova.compute.manager [-] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 741.611213] env[62499]: DEBUG nova.network.neutron [-] [instance: 78a83324-36da-457e-a78d-c82d0a722015] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 741.633965] env[62499]: DEBUG nova.network.neutron [-] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 741.647608] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.675s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 741.648178] env[62499]: DEBUG nova.compute.manager [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 741.651079] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.500s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 741.652452] env[62499]: INFO nova.compute.claims [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 741.851685] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Acquiring lock "20e4ff1b-0b84-477d-a0d0-a85a439449ad" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 741.851959] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Lock "20e4ff1b-0b84-477d-a0d0-a85a439449ad" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 742.136024] env[62499]: DEBUG nova.network.neutron [-] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 742.162647] env[62499]: DEBUG nova.compute.utils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 742.165189] env[62499]: DEBUG nova.compute.manager [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 742.165360] env[62499]: DEBUG nova.network.neutron [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 742.205992] env[62499]: INFO nova.scheduler.client.report [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Deleted allocations for instance 6d8b7855-567c-44c3-90ee-fc0daec87f46 [ 742.225286] env[62499]: DEBUG nova.policy [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '9ad60a9ead734f08aa163c15c9a6b81d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9c30565c40a845588906f604e8cb0e00', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 742.251180] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 742.251180] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 742.501519] env[62499]: DEBUG nova.network.neutron [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Successfully created port: 724ced36-552b-431b-84db-b53ae96f6fda {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 742.638930] env[62499]: INFO nova.compute.manager [-] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Took 1.03 seconds to deallocate network for instance. [ 742.643252] env[62499]: DEBUG nova.compute.claims [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 742.643533] env[62499]: DEBUG oslo_concurrency.lockutils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 742.666332] env[62499]: DEBUG nova.compute.manager [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 742.717327] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dc9bd693-02cd-4f3d-ba3c-be92e83d1e51 tempest-ServersV294TestFqdnHostnames-987785393 tempest-ServersV294TestFqdnHostnames-987785393-project-member] Lock "6d8b7855-567c-44c3-90ee-fc0daec87f46" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 125.986s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 742.760980] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 742.761245] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Starting heal instance info cache {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9928}} [ 742.761417] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Rebuilding the list of instances to heal {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 742.978353] env[62499]: DEBUG nova.network.neutron [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Successfully created port: dd417757-f08e-4643-9132-8caf7c922579 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 743.075932] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e91f20fa-e49f-49d6-ba3b-ccad53e1fb1e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.084200] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-89b6f27e-8925-41fc-aa77-3616ac200c0e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.116391] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-265de974-92c4-4a8a-87de-1e21f8c64f01 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.123716] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-645bcf3d-eedb-4f95-a5be-849965200d56 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.136901] env[62499]: DEBUG nova.compute.provider_tree [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 743.220501] env[62499]: DEBUG nova.compute.manager [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 743.264530] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 743.264699] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 743.264831] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 743.264954] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 743.265091] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 743.265216] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Didn't find any instances for network info cache update. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10014}} [ 743.265656] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 743.265827] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 743.265972] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 743.266153] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 743.266296] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 743.266778] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 743.266778] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62499) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10547}} [ 743.266778] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager.update_available_resource {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 743.640329] env[62499]: DEBUG nova.scheduler.client.report [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 743.683013] env[62499]: DEBUG nova.compute.manager [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 743.717260] env[62499]: DEBUG nova.virt.hardware [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 743.717260] env[62499]: DEBUG nova.virt.hardware [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 743.717260] env[62499]: DEBUG nova.virt.hardware [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 743.717260] env[62499]: DEBUG nova.virt.hardware [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 743.717443] env[62499]: DEBUG nova.virt.hardware [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 743.717443] env[62499]: DEBUG nova.virt.hardware [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 743.717604] env[62499]: DEBUG nova.virt.hardware [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 743.717754] env[62499]: DEBUG nova.virt.hardware [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 743.719056] env[62499]: DEBUG nova.virt.hardware [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 743.719056] env[62499]: DEBUG nova.virt.hardware [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 743.719056] env[62499]: DEBUG nova.virt.hardware [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 743.720378] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3978caef-ba80-4716-9d99-c10648ff7601 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.732091] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d467804-8486-4d64-a81e-db0ebe683f89 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.760065] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 743.769326] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 744.147647] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.496s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 744.148234] env[62499]: DEBUG nova.compute.manager [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 744.151449] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.895s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 744.152922] env[62499]: INFO nova.compute.claims [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 744.617499] env[62499]: DEBUG nova.compute.manager [req-bd122dac-401c-4331-92e5-873edf68126d req-0185ce49-54da-44f6-bc0e-12eeae5fd2e3 service nova] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Received event network-changed-724ced36-552b-431b-84db-b53ae96f6fda {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 744.617499] env[62499]: DEBUG nova.compute.manager [req-bd122dac-401c-4331-92e5-873edf68126d req-0185ce49-54da-44f6-bc0e-12eeae5fd2e3 service nova] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Refreshing instance network info cache due to event network-changed-724ced36-552b-431b-84db-b53ae96f6fda. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 744.617818] env[62499]: DEBUG oslo_concurrency.lockutils [req-bd122dac-401c-4331-92e5-873edf68126d req-0185ce49-54da-44f6-bc0e-12eeae5fd2e3 service nova] Acquiring lock "refresh_cache-dee17e73-22db-48ff-b0b0-ec7bec850c48" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 744.617975] env[62499]: DEBUG oslo_concurrency.lockutils [req-bd122dac-401c-4331-92e5-873edf68126d req-0185ce49-54da-44f6-bc0e-12eeae5fd2e3 service nova] Acquired lock "refresh_cache-dee17e73-22db-48ff-b0b0-ec7bec850c48" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 744.618882] env[62499]: DEBUG nova.network.neutron [req-bd122dac-401c-4331-92e5-873edf68126d req-0185ce49-54da-44f6-bc0e-12eeae5fd2e3 service nova] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Refreshing network info cache for port 724ced36-552b-431b-84db-b53ae96f6fda {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 744.669880] env[62499]: DEBUG nova.compute.utils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 744.673949] env[62499]: DEBUG nova.compute.manager [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 744.673949] env[62499]: DEBUG nova.network.neutron [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 744.739512] env[62499]: DEBUG nova.policy [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a15aadeb30a04c09b35aae1661b89b66', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '076da2170578492082394cfcb4b102b5', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 744.910674] env[62499]: ERROR nova.compute.manager [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 724ced36-552b-431b-84db-b53ae96f6fda, please check neutron logs for more information. [ 744.910674] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 744.910674] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 744.910674] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 744.910674] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 744.910674] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 744.910674] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 744.910674] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 744.910674] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 744.910674] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 744.910674] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 744.910674] env[62499]: ERROR nova.compute.manager raise self.value [ 744.910674] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 744.910674] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 744.910674] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 744.910674] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 744.911256] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 744.911256] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 744.911256] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 724ced36-552b-431b-84db-b53ae96f6fda, please check neutron logs for more information. [ 744.911256] env[62499]: ERROR nova.compute.manager [ 744.911256] env[62499]: Traceback (most recent call last): [ 744.911256] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 744.911256] env[62499]: listener.cb(fileno) [ 744.911256] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 744.911256] env[62499]: result = function(*args, **kwargs) [ 744.911256] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 744.911256] env[62499]: return func(*args, **kwargs) [ 744.911256] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 744.911256] env[62499]: raise e [ 744.911256] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 744.911256] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 744.911256] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 744.911256] env[62499]: created_port_ids = self._update_ports_for_instance( [ 744.911256] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 744.911256] env[62499]: with excutils.save_and_reraise_exception(): [ 744.911256] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 744.911256] env[62499]: self.force_reraise() [ 744.911256] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 744.911256] env[62499]: raise self.value [ 744.911256] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 744.911256] env[62499]: updated_port = self._update_port( [ 744.911256] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 744.911256] env[62499]: _ensure_no_port_binding_failure(port) [ 744.911256] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 744.911256] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 744.912109] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 724ced36-552b-431b-84db-b53ae96f6fda, please check neutron logs for more information. [ 744.912109] env[62499]: Removing descriptor: 16 [ 744.912169] env[62499]: ERROR nova.compute.manager [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 724ced36-552b-431b-84db-b53ae96f6fda, please check neutron logs for more information. [ 744.912169] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Traceback (most recent call last): [ 744.912169] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 744.912169] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] yield resources [ 744.912169] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 744.912169] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] self.driver.spawn(context, instance, image_meta, [ 744.912169] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 744.912169] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] self._vmops.spawn(context, instance, image_meta, injected_files, [ 744.912169] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 744.912169] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] vm_ref = self.build_virtual_machine(instance, [ 744.912169] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 744.912493] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] vif_infos = vmwarevif.get_vif_info(self._session, [ 744.912493] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 744.912493] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] for vif in network_info: [ 744.912493] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 744.912493] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] return self._sync_wrapper(fn, *args, **kwargs) [ 744.912493] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 744.912493] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] self.wait() [ 744.912493] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 744.912493] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] self[:] = self._gt.wait() [ 744.912493] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 744.912493] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] return self._exit_event.wait() [ 744.912493] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 744.912493] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] result = hub.switch() [ 744.912915] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 744.912915] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] return self.greenlet.switch() [ 744.912915] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 744.912915] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] result = function(*args, **kwargs) [ 744.912915] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 744.912915] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] return func(*args, **kwargs) [ 744.912915] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 744.912915] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] raise e [ 744.912915] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 744.912915] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] nwinfo = self.network_api.allocate_for_instance( [ 744.912915] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 744.912915] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] created_port_ids = self._update_ports_for_instance( [ 744.912915] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 744.913325] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] with excutils.save_and_reraise_exception(): [ 744.913325] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 744.913325] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] self.force_reraise() [ 744.913325] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 744.913325] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] raise self.value [ 744.913325] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 744.913325] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] updated_port = self._update_port( [ 744.913325] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 744.913325] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] _ensure_no_port_binding_failure(port) [ 744.913325] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 744.913325] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] raise exception.PortBindingFailed(port_id=port['id']) [ 744.913325] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] nova.exception.PortBindingFailed: Binding failed for port 724ced36-552b-431b-84db-b53ae96f6fda, please check neutron logs for more information. [ 744.913325] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] [ 744.913707] env[62499]: INFO nova.compute.manager [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Terminating instance [ 744.916393] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Acquiring lock "refresh_cache-dee17e73-22db-48ff-b0b0-ec7bec850c48" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 745.176350] env[62499]: DEBUG nova.network.neutron [req-bd122dac-401c-4331-92e5-873edf68126d req-0185ce49-54da-44f6-bc0e-12eeae5fd2e3 service nova] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 745.176350] env[62499]: DEBUG nova.compute.manager [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 745.279702] env[62499]: DEBUG nova.network.neutron [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Successfully created port: f2efa323-2a49-4acf-84ce-6cd3af0dcd52 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 745.327309] env[62499]: DEBUG nova.network.neutron [req-bd122dac-401c-4331-92e5-873edf68126d req-0185ce49-54da-44f6-bc0e-12eeae5fd2e3 service nova] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 745.656253] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c5a29be1-56d8-4052-b49e-edb3ad9da351 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.664918] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3489fdf6-4587-4bdb-9f6c-c69647369cbe {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.713863] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4ce49c2-48a4-4f7d-9ad7-07f6d13cc8e8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.726392] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2845b991-ec99-4a92-a1e1-144746a1b295 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.741411] env[62499]: DEBUG nova.compute.provider_tree [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 745.829939] env[62499]: DEBUG oslo_concurrency.lockutils [req-bd122dac-401c-4331-92e5-873edf68126d req-0185ce49-54da-44f6-bc0e-12eeae5fd2e3 service nova] Releasing lock "refresh_cache-dee17e73-22db-48ff-b0b0-ec7bec850c48" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 745.830566] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Acquired lock "refresh_cache-dee17e73-22db-48ff-b0b0-ec7bec850c48" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 745.830566] env[62499]: DEBUG nova.network.neutron [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 746.221182] env[62499]: DEBUG nova.compute.manager [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 746.247063] env[62499]: DEBUG nova.scheduler.client.report [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 746.256341] env[62499]: DEBUG nova.virt.hardware [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 746.257078] env[62499]: DEBUG nova.virt.hardware [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 746.257078] env[62499]: DEBUG nova.virt.hardware [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 746.257078] env[62499]: DEBUG nova.virt.hardware [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 746.257078] env[62499]: DEBUG nova.virt.hardware [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 746.257078] env[62499]: DEBUG nova.virt.hardware [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 746.257577] env[62499]: DEBUG nova.virt.hardware [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 746.257577] env[62499]: DEBUG nova.virt.hardware [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 746.257577] env[62499]: DEBUG nova.virt.hardware [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 746.258951] env[62499]: DEBUG nova.virt.hardware [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 746.258951] env[62499]: DEBUG nova.virt.hardware [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 746.258951] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d6b9cd1-8d7e-4ac4-83de-72ec2eb64c2b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.270581] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29173534-250f-4de6-a3fd-d9c29dcb9214 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.368646] env[62499]: DEBUG nova.compute.manager [req-a33ca3b0-c365-4a3a-bad2-59a275e549dd req-af52ab8b-6146-41e9-ba7e-416b22b1549b service nova] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Received event network-changed-f2efa323-2a49-4acf-84ce-6cd3af0dcd52 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 746.368838] env[62499]: DEBUG nova.compute.manager [req-a33ca3b0-c365-4a3a-bad2-59a275e549dd req-af52ab8b-6146-41e9-ba7e-416b22b1549b service nova] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Refreshing instance network info cache due to event network-changed-f2efa323-2a49-4acf-84ce-6cd3af0dcd52. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 746.369319] env[62499]: DEBUG oslo_concurrency.lockutils [req-a33ca3b0-c365-4a3a-bad2-59a275e549dd req-af52ab8b-6146-41e9-ba7e-416b22b1549b service nova] Acquiring lock "refresh_cache-f5b10ac7-e20c-439c-bc6f-a3cf2dd26122" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 746.369541] env[62499]: DEBUG oslo_concurrency.lockutils [req-a33ca3b0-c365-4a3a-bad2-59a275e549dd req-af52ab8b-6146-41e9-ba7e-416b22b1549b service nova] Acquired lock "refresh_cache-f5b10ac7-e20c-439c-bc6f-a3cf2dd26122" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 746.369910] env[62499]: DEBUG nova.network.neutron [req-a33ca3b0-c365-4a3a-bad2-59a275e549dd req-af52ab8b-6146-41e9-ba7e-416b22b1549b service nova] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Refreshing network info cache for port f2efa323-2a49-4acf-84ce-6cd3af0dcd52 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 746.380867] env[62499]: DEBUG nova.network.neutron [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 746.469423] env[62499]: ERROR nova.compute.manager [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f2efa323-2a49-4acf-84ce-6cd3af0dcd52, please check neutron logs for more information. [ 746.469423] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 746.469423] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 746.469423] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 746.469423] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 746.469423] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 746.469423] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 746.469423] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 746.469423] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 746.469423] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 746.469423] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 746.469423] env[62499]: ERROR nova.compute.manager raise self.value [ 746.469423] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 746.469423] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 746.469423] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 746.469423] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 746.469960] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 746.469960] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 746.469960] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f2efa323-2a49-4acf-84ce-6cd3af0dcd52, please check neutron logs for more information. [ 746.469960] env[62499]: ERROR nova.compute.manager [ 746.469960] env[62499]: Traceback (most recent call last): [ 746.469960] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 746.469960] env[62499]: listener.cb(fileno) [ 746.469960] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 746.469960] env[62499]: result = function(*args, **kwargs) [ 746.469960] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 746.469960] env[62499]: return func(*args, **kwargs) [ 746.469960] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 746.469960] env[62499]: raise e [ 746.469960] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 746.469960] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 746.469960] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 746.469960] env[62499]: created_port_ids = self._update_ports_for_instance( [ 746.469960] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 746.469960] env[62499]: with excutils.save_and_reraise_exception(): [ 746.469960] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 746.469960] env[62499]: self.force_reraise() [ 746.469960] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 746.469960] env[62499]: raise self.value [ 746.469960] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 746.469960] env[62499]: updated_port = self._update_port( [ 746.469960] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 746.469960] env[62499]: _ensure_no_port_binding_failure(port) [ 746.469960] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 746.469960] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 746.470838] env[62499]: nova.exception.PortBindingFailed: Binding failed for port f2efa323-2a49-4acf-84ce-6cd3af0dcd52, please check neutron logs for more information. [ 746.470838] env[62499]: Removing descriptor: 17 [ 746.470838] env[62499]: ERROR nova.compute.manager [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f2efa323-2a49-4acf-84ce-6cd3af0dcd52, please check neutron logs for more information. [ 746.470838] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Traceback (most recent call last): [ 746.470838] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 746.470838] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] yield resources [ 746.470838] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 746.470838] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] self.driver.spawn(context, instance, image_meta, [ 746.470838] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 746.470838] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] self._vmops.spawn(context, instance, image_meta, injected_files, [ 746.470838] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 746.470838] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] vm_ref = self.build_virtual_machine(instance, [ 746.471222] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 746.471222] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] vif_infos = vmwarevif.get_vif_info(self._session, [ 746.471222] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 746.471222] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] for vif in network_info: [ 746.471222] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 746.471222] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] return self._sync_wrapper(fn, *args, **kwargs) [ 746.471222] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 746.471222] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] self.wait() [ 746.471222] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 746.471222] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] self[:] = self._gt.wait() [ 746.471222] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 746.471222] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] return self._exit_event.wait() [ 746.471222] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 746.471634] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] result = hub.switch() [ 746.471634] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 746.471634] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] return self.greenlet.switch() [ 746.471634] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 746.471634] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] result = function(*args, **kwargs) [ 746.471634] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 746.471634] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] return func(*args, **kwargs) [ 746.471634] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 746.471634] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] raise e [ 746.471634] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 746.471634] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] nwinfo = self.network_api.allocate_for_instance( [ 746.471634] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 746.471634] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] created_port_ids = self._update_ports_for_instance( [ 746.472024] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 746.472024] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] with excutils.save_and_reraise_exception(): [ 746.472024] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 746.472024] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] self.force_reraise() [ 746.472024] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 746.472024] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] raise self.value [ 746.472024] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 746.472024] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] updated_port = self._update_port( [ 746.472024] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 746.472024] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] _ensure_no_port_binding_failure(port) [ 746.472024] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 746.472024] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] raise exception.PortBindingFailed(port_id=port['id']) [ 746.472455] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] nova.exception.PortBindingFailed: Binding failed for port f2efa323-2a49-4acf-84ce-6cd3af0dcd52, please check neutron logs for more information. [ 746.472455] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] [ 746.472455] env[62499]: INFO nova.compute.manager [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Terminating instance [ 746.472909] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Acquiring lock "refresh_cache-f5b10ac7-e20c-439c-bc6f-a3cf2dd26122" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 746.525697] env[62499]: DEBUG nova.network.neutron [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 746.727125] env[62499]: DEBUG nova.compute.manager [req-dc187f1c-8db2-4393-9c45-7c0a7cd173a1 req-1b40b3e9-81d3-4b8f-9229-103c87711ef0 service nova] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Received event network-vif-deleted-724ced36-552b-431b-84db-b53ae96f6fda {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 746.765021] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.611s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 746.765021] env[62499]: DEBUG nova.compute.manager [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 746.766184] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.423s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 746.841957] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Acquiring lock "4eda80db-5db7-4fd6-9932-108c22f4a616" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 746.842213] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Lock "4eda80db-5db7-4fd6-9932-108c22f4a616" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 746.889539] env[62499]: DEBUG nova.network.neutron [req-a33ca3b0-c365-4a3a-bad2-59a275e549dd req-af52ab8b-6146-41e9-ba7e-416b22b1549b service nova] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 746.982108] env[62499]: DEBUG nova.network.neutron [req-a33ca3b0-c365-4a3a-bad2-59a275e549dd req-af52ab8b-6146-41e9-ba7e-416b22b1549b service nova] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 747.027830] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Releasing lock "refresh_cache-dee17e73-22db-48ff-b0b0-ec7bec850c48" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 747.028279] env[62499]: DEBUG nova.compute.manager [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 747.028474] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 747.028755] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-cfb85c03-66b4-4ee0-9aca-3b62fd302746 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 747.038127] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-abc2c2cd-ae4c-4153-b546-ad6a4a22462c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 747.061237] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance dee17e73-22db-48ff-b0b0-ec7bec850c48 could not be found. [ 747.061469] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 747.061676] env[62499]: INFO nova.compute.manager [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Took 0.03 seconds to destroy the instance on the hypervisor. [ 747.061921] env[62499]: DEBUG oslo.service.loopingcall [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 747.062136] env[62499]: DEBUG nova.compute.manager [-] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 747.062231] env[62499]: DEBUG nova.network.neutron [-] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 747.105056] env[62499]: DEBUG nova.network.neutron [-] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 747.270491] env[62499]: DEBUG nova.compute.utils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 747.276772] env[62499]: DEBUG nova.compute.manager [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 747.276974] env[62499]: DEBUG nova.network.neutron [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 747.324017] env[62499]: DEBUG nova.policy [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a15aadeb30a04c09b35aae1661b89b66', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '076da2170578492082394cfcb4b102b5', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 747.486455] env[62499]: DEBUG oslo_concurrency.lockutils [req-a33ca3b0-c365-4a3a-bad2-59a275e549dd req-af52ab8b-6146-41e9-ba7e-416b22b1549b service nova] Releasing lock "refresh_cache-f5b10ac7-e20c-439c-bc6f-a3cf2dd26122" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 747.488965] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Acquired lock "refresh_cache-f5b10ac7-e20c-439c-bc6f-a3cf2dd26122" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 747.488965] env[62499]: DEBUG nova.network.neutron [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 747.683391] env[62499]: DEBUG nova.network.neutron [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Successfully created port: 9cdf2850-ba3f-49bf-afc4-6b926de1325b {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 747.692028] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e972a7a-e87d-4b57-86c7-e4388670244c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 747.699370] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ea74ba7-3d4d-4b8a-9b58-9726b0372721 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 747.735358] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9cfb539d-a4e5-4d25-8f00-6b46d34d9905 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 747.743346] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-71a14c47-c186-4e7d-b80a-6969bd157c75 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 747.756393] env[62499]: DEBUG nova.compute.provider_tree [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 747.777205] env[62499]: DEBUG nova.compute.manager [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 747.832909] env[62499]: DEBUG nova.network.neutron [-] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 747.928157] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquiring lock "c1961f60-06d8-44d8-999f-d498c3667adc" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 747.928743] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "c1961f60-06d8-44d8-999f-d498c3667adc" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 748.007972] env[62499]: DEBUG nova.network.neutron [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 748.099844] env[62499]: DEBUG nova.network.neutron [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 748.245352] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquiring lock "ba8fc3e0-dae8-4910-858b-dab1de72089e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 748.245597] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "ba8fc3e0-dae8-4910-858b-dab1de72089e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 748.259105] env[62499]: DEBUG nova.scheduler.client.report [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 748.337428] env[62499]: INFO nova.compute.manager [-] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Took 1.28 seconds to deallocate network for instance. [ 748.340155] env[62499]: DEBUG nova.compute.claims [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 748.340292] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 748.395770] env[62499]: DEBUG nova.compute.manager [req-952cb468-f1c9-40a7-8313-425f1134cec4 req-7c1f409f-51dc-4d27-a9b0-d8041f912acc service nova] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Received event network-vif-deleted-f2efa323-2a49-4acf-84ce-6cd3af0dcd52 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 748.602314] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Releasing lock "refresh_cache-f5b10ac7-e20c-439c-bc6f-a3cf2dd26122" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 748.602743] env[62499]: DEBUG nova.compute.manager [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 748.602948] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 748.603253] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1b63705b-e71c-4a26-8058-6319b1cd00ff {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.611792] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-161bf724-3094-44ed-985d-d227021737f4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.632290] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance f5b10ac7-e20c-439c-bc6f-a3cf2dd26122 could not be found. [ 748.632527] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 748.632705] env[62499]: INFO nova.compute.manager [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Took 0.03 seconds to destroy the instance on the hypervisor. [ 748.632948] env[62499]: DEBUG oslo.service.loopingcall [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 748.633176] env[62499]: DEBUG nova.compute.manager [-] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 748.633262] env[62499]: DEBUG nova.network.neutron [-] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 748.647927] env[62499]: DEBUG nova.network.neutron [-] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 748.746063] env[62499]: ERROR nova.compute.manager [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 9cdf2850-ba3f-49bf-afc4-6b926de1325b, please check neutron logs for more information. [ 748.746063] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 748.746063] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 748.746063] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 748.746063] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 748.746063] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 748.746063] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 748.746063] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 748.746063] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 748.746063] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 748.746063] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 748.746063] env[62499]: ERROR nova.compute.manager raise self.value [ 748.746063] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 748.746063] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 748.746063] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 748.746063] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 748.746806] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 748.746806] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 748.746806] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 9cdf2850-ba3f-49bf-afc4-6b926de1325b, please check neutron logs for more information. [ 748.746806] env[62499]: ERROR nova.compute.manager [ 748.746806] env[62499]: Traceback (most recent call last): [ 748.746806] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 748.746806] env[62499]: listener.cb(fileno) [ 748.746806] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 748.746806] env[62499]: result = function(*args, **kwargs) [ 748.746806] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 748.746806] env[62499]: return func(*args, **kwargs) [ 748.746806] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 748.746806] env[62499]: raise e [ 748.746806] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 748.746806] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 748.746806] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 748.746806] env[62499]: created_port_ids = self._update_ports_for_instance( [ 748.746806] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 748.746806] env[62499]: with excutils.save_and_reraise_exception(): [ 748.746806] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 748.746806] env[62499]: self.force_reraise() [ 748.746806] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 748.746806] env[62499]: raise self.value [ 748.746806] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 748.746806] env[62499]: updated_port = self._update_port( [ 748.746806] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 748.746806] env[62499]: _ensure_no_port_binding_failure(port) [ 748.746806] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 748.746806] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 748.747711] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 9cdf2850-ba3f-49bf-afc4-6b926de1325b, please check neutron logs for more information. [ 748.747711] env[62499]: Removing descriptor: 17 [ 748.764227] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.998s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 748.764677] env[62499]: ERROR nova.compute.manager [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 9625f688-ed1b-4676-a575-789088c0a6e6, please check neutron logs for more information. [ 748.764677] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Traceback (most recent call last): [ 748.764677] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 748.764677] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] self.driver.spawn(context, instance, image_meta, [ 748.764677] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 748.764677] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 748.764677] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 748.764677] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] vm_ref = self.build_virtual_machine(instance, [ 748.764677] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 748.764677] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] vif_infos = vmwarevif.get_vif_info(self._session, [ 748.764677] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 748.765052] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] for vif in network_info: [ 748.765052] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 748.765052] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] return self._sync_wrapper(fn, *args, **kwargs) [ 748.765052] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 748.765052] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] self.wait() [ 748.765052] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 748.765052] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] self[:] = self._gt.wait() [ 748.765052] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 748.765052] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] return self._exit_event.wait() [ 748.765052] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 748.765052] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] result = hub.switch() [ 748.765052] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 748.765052] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] return self.greenlet.switch() [ 748.765409] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 748.765409] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] result = function(*args, **kwargs) [ 748.765409] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 748.765409] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] return func(*args, **kwargs) [ 748.765409] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 748.765409] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] raise e [ 748.765409] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 748.765409] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] nwinfo = self.network_api.allocate_for_instance( [ 748.765409] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 748.765409] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] created_port_ids = self._update_ports_for_instance( [ 748.765409] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 748.765409] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] with excutils.save_and_reraise_exception(): [ 748.765409] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 748.765770] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] self.force_reraise() [ 748.765770] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 748.765770] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] raise self.value [ 748.765770] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 748.765770] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] updated_port = self._update_port( [ 748.765770] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 748.765770] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] _ensure_no_port_binding_failure(port) [ 748.765770] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 748.765770] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] raise exception.PortBindingFailed(port_id=port['id']) [ 748.765770] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] nova.exception.PortBindingFailed: Binding failed for port 9625f688-ed1b-4676-a575-789088c0a6e6, please check neutron logs for more information. [ 748.765770] env[62499]: ERROR nova.compute.manager [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] [ 748.766093] env[62499]: DEBUG nova.compute.utils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Binding failed for port 9625f688-ed1b-4676-a575-789088c0a6e6, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 748.766719] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.034s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 748.768533] env[62499]: INFO nova.compute.claims [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 748.771197] env[62499]: DEBUG nova.compute.manager [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Build of instance 8a769120-906c-4e7c-ac19-b7895039efc1 was re-scheduled: Binding failed for port 9625f688-ed1b-4676-a575-789088c0a6e6, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 748.771646] env[62499]: DEBUG nova.compute.manager [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 748.771951] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Acquiring lock "refresh_cache-8a769120-906c-4e7c-ac19-b7895039efc1" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 748.772140] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Acquired lock "refresh_cache-8a769120-906c-4e7c-ac19-b7895039efc1" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 748.772303] env[62499]: DEBUG nova.network.neutron [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 748.786079] env[62499]: DEBUG nova.compute.manager [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 748.820142] env[62499]: DEBUG nova.virt.hardware [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 748.820515] env[62499]: DEBUG nova.virt.hardware [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 748.820603] env[62499]: DEBUG nova.virt.hardware [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 748.820728] env[62499]: DEBUG nova.virt.hardware [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 748.820867] env[62499]: DEBUG nova.virt.hardware [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 748.821021] env[62499]: DEBUG nova.virt.hardware [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 748.821782] env[62499]: DEBUG nova.virt.hardware [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 748.821782] env[62499]: DEBUG nova.virt.hardware [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 748.821782] env[62499]: DEBUG nova.virt.hardware [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 748.822034] env[62499]: DEBUG nova.virt.hardware [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 748.822257] env[62499]: DEBUG nova.virt.hardware [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 748.823450] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69a6f2b0-a3cd-4361-8f99-ab6802480f05 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.831698] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86044f9c-8e43-4b3d-9fd8-eea013c7938d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.847111] env[62499]: ERROR nova.compute.manager [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 9cdf2850-ba3f-49bf-afc4-6b926de1325b, please check neutron logs for more information. [ 748.847111] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Traceback (most recent call last): [ 748.847111] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 748.847111] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] yield resources [ 748.847111] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 748.847111] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] self.driver.spawn(context, instance, image_meta, [ 748.847111] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 748.847111] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 748.847111] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 748.847111] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] vm_ref = self.build_virtual_machine(instance, [ 748.847111] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 748.847558] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] vif_infos = vmwarevif.get_vif_info(self._session, [ 748.847558] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 748.847558] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] for vif in network_info: [ 748.847558] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 748.847558] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] return self._sync_wrapper(fn, *args, **kwargs) [ 748.847558] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 748.847558] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] self.wait() [ 748.847558] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 748.847558] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] self[:] = self._gt.wait() [ 748.847558] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 748.847558] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] return self._exit_event.wait() [ 748.847558] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 748.847558] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] current.throw(*self._exc) [ 748.847995] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 748.847995] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] result = function(*args, **kwargs) [ 748.847995] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 748.847995] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] return func(*args, **kwargs) [ 748.847995] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 748.847995] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] raise e [ 748.847995] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 748.847995] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] nwinfo = self.network_api.allocate_for_instance( [ 748.847995] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 748.847995] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] created_port_ids = self._update_ports_for_instance( [ 748.847995] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 748.847995] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] with excutils.save_and_reraise_exception(): [ 748.847995] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 748.848506] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] self.force_reraise() [ 748.848506] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 748.848506] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] raise self.value [ 748.848506] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 748.848506] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] updated_port = self._update_port( [ 748.848506] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 748.848506] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] _ensure_no_port_binding_failure(port) [ 748.848506] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 748.848506] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] raise exception.PortBindingFailed(port_id=port['id']) [ 748.848506] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] nova.exception.PortBindingFailed: Binding failed for port 9cdf2850-ba3f-49bf-afc4-6b926de1325b, please check neutron logs for more information. [ 748.848506] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] [ 748.848506] env[62499]: INFO nova.compute.manager [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Terminating instance [ 748.849868] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Acquiring lock "refresh_cache-ff822880-41dc-429f-80fb-a1ddc0441ea3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 748.850040] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Acquired lock "refresh_cache-ff822880-41dc-429f-80fb-a1ddc0441ea3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 748.850208] env[62499]: DEBUG nova.network.neutron [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 748.883470] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquiring lock "0eaadb76-e2c7-4e06-b865-065ab0f014ba" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 748.883707] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "0eaadb76-e2c7-4e06-b865-065ab0f014ba" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 749.150538] env[62499]: DEBUG nova.network.neutron [-] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 749.291426] env[62499]: DEBUG nova.network.neutron [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 749.367025] env[62499]: DEBUG nova.network.neutron [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 749.369545] env[62499]: DEBUG nova.network.neutron [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 749.437285] env[62499]: DEBUG nova.network.neutron [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 749.655643] env[62499]: INFO nova.compute.manager [-] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Took 1.02 seconds to deallocate network for instance. [ 749.658227] env[62499]: DEBUG nova.compute.claims [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 749.658411] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 749.873435] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Releasing lock "refresh_cache-8a769120-906c-4e7c-ac19-b7895039efc1" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 749.874381] env[62499]: DEBUG nova.compute.manager [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 749.874755] env[62499]: DEBUG nova.compute.manager [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 749.875196] env[62499]: DEBUG nova.network.neutron [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 749.890177] env[62499]: DEBUG nova.network.neutron [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 749.940810] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Releasing lock "refresh_cache-ff822880-41dc-429f-80fb-a1ddc0441ea3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 749.941278] env[62499]: DEBUG nova.compute.manager [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 749.941436] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 749.941958] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c247ca53-03c4-4486-bee1-a28bfa341917 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 749.950822] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef3ad508-cd16-4274-ae07-18c514a67c51 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 749.974122] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ff822880-41dc-429f-80fb-a1ddc0441ea3 could not be found. [ 749.974399] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 749.974614] env[62499]: INFO nova.compute.manager [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Took 0.03 seconds to destroy the instance on the hypervisor. [ 749.974887] env[62499]: DEBUG oslo.service.loopingcall [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 749.977211] env[62499]: DEBUG nova.compute.manager [-] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 749.977305] env[62499]: DEBUG nova.network.neutron [-] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 750.001258] env[62499]: DEBUG nova.network.neutron [-] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 750.177690] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8bb8369-84c3-43d5-8642-402f18ed3bf6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.185502] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b45d4aa4-6dd6-4d67-b95f-95c2c32a99d5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.215185] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d37bb450-f952-4034-9803-a8f9d742e7b7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.222429] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d72add35-f7c6-42e0-b5b3-bf673172278b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.235899] env[62499]: DEBUG nova.compute.provider_tree [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 750.392894] env[62499]: DEBUG nova.network.neutron [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 750.420112] env[62499]: DEBUG nova.compute.manager [req-a7ded8fc-8575-40d6-96ae-67afc051c0ae req-4f208df4-ad02-40c1-b339-e92706d87f0f service nova] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Received event network-changed-9cdf2850-ba3f-49bf-afc4-6b926de1325b {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 750.420225] env[62499]: DEBUG nova.compute.manager [req-a7ded8fc-8575-40d6-96ae-67afc051c0ae req-4f208df4-ad02-40c1-b339-e92706d87f0f service nova] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Refreshing instance network info cache due to event network-changed-9cdf2850-ba3f-49bf-afc4-6b926de1325b. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 750.420440] env[62499]: DEBUG oslo_concurrency.lockutils [req-a7ded8fc-8575-40d6-96ae-67afc051c0ae req-4f208df4-ad02-40c1-b339-e92706d87f0f service nova] Acquiring lock "refresh_cache-ff822880-41dc-429f-80fb-a1ddc0441ea3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 750.420583] env[62499]: DEBUG oslo_concurrency.lockutils [req-a7ded8fc-8575-40d6-96ae-67afc051c0ae req-4f208df4-ad02-40c1-b339-e92706d87f0f service nova] Acquired lock "refresh_cache-ff822880-41dc-429f-80fb-a1ddc0441ea3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 750.420743] env[62499]: DEBUG nova.network.neutron [req-a7ded8fc-8575-40d6-96ae-67afc051c0ae req-4f208df4-ad02-40c1-b339-e92706d87f0f service nova] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Refreshing network info cache for port 9cdf2850-ba3f-49bf-afc4-6b926de1325b {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 750.503790] env[62499]: DEBUG nova.network.neutron [-] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 750.739708] env[62499]: DEBUG nova.scheduler.client.report [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 750.895163] env[62499]: INFO nova.compute.manager [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] [instance: 8a769120-906c-4e7c-ac19-b7895039efc1] Took 1.02 seconds to deallocate network for instance. [ 750.939691] env[62499]: DEBUG nova.network.neutron [req-a7ded8fc-8575-40d6-96ae-67afc051c0ae req-4f208df4-ad02-40c1-b339-e92706d87f0f service nova] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 751.006406] env[62499]: INFO nova.compute.manager [-] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Took 1.03 seconds to deallocate network for instance. [ 751.008659] env[62499]: DEBUG nova.compute.claims [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 751.008847] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 751.022460] env[62499]: DEBUG nova.network.neutron [req-a7ded8fc-8575-40d6-96ae-67afc051c0ae req-4f208df4-ad02-40c1-b339-e92706d87f0f service nova] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 751.244514] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.477s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 751.244809] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 751.247322] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.504s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 751.249073] env[62499]: INFO nova.compute.claims [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 751.528778] env[62499]: DEBUG oslo_concurrency.lockutils [req-a7ded8fc-8575-40d6-96ae-67afc051c0ae req-4f208df4-ad02-40c1-b339-e92706d87f0f service nova] Releasing lock "refresh_cache-ff822880-41dc-429f-80fb-a1ddc0441ea3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 751.529133] env[62499]: DEBUG nova.compute.manager [req-a7ded8fc-8575-40d6-96ae-67afc051c0ae req-4f208df4-ad02-40c1-b339-e92706d87f0f service nova] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Received event network-vif-deleted-9cdf2850-ba3f-49bf-afc4-6b926de1325b {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 751.753512] env[62499]: DEBUG nova.compute.utils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 751.757202] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 751.757372] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 751.808155] env[62499]: DEBUG nova.policy [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2aae85566cb84e41881222ddaa137bd5', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'dff7a8db03474072848c122886f622d7', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 751.932819] env[62499]: INFO nova.scheduler.client.report [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Deleted allocations for instance 8a769120-906c-4e7c-ac19-b7895039efc1 [ 752.258586] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 752.277504] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Successfully created port: 0a9d41a4-fdd1-4ccc-82a7-302751c3e083 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 752.441243] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be438baf-e4ff-4f8b-aca3-6b1d91343a65 tempest-ServerPasswordTestJSON-2127054147 tempest-ServerPasswordTestJSON-2127054147-project-member] Lock "8a769120-906c-4e7c-ac19-b7895039efc1" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 133.162s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.652151] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-24625334-06db-4779-abce-c48b9fb260b6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 752.659582] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f09e4bfe-7671-4650-a8ef-4416571f08bd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 752.688618] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6619ebe4-0fc3-44b8-8a38-e24bcc6d0a1d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 752.696316] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2009f636-423f-49f4-a251-d057be682d46 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 752.713777] env[62499]: DEBUG nova.compute.provider_tree [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 752.949555] env[62499]: DEBUG nova.compute.manager [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 753.054107] env[62499]: DEBUG nova.compute.manager [req-751defba-a26b-4775-b8b0-9bd7d09c3591 req-70ef2422-ca0a-4637-90f7-05ebebf5e4b5 service nova] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Received event network-changed-0a9d41a4-fdd1-4ccc-82a7-302751c3e083 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 753.054315] env[62499]: DEBUG nova.compute.manager [req-751defba-a26b-4775-b8b0-9bd7d09c3591 req-70ef2422-ca0a-4637-90f7-05ebebf5e4b5 service nova] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Refreshing instance network info cache due to event network-changed-0a9d41a4-fdd1-4ccc-82a7-302751c3e083. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 753.054535] env[62499]: DEBUG oslo_concurrency.lockutils [req-751defba-a26b-4775-b8b0-9bd7d09c3591 req-70ef2422-ca0a-4637-90f7-05ebebf5e4b5 service nova] Acquiring lock "refresh_cache-0224e4da-3079-477f-96d8-671985e85c39" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 753.054665] env[62499]: DEBUG oslo_concurrency.lockutils [req-751defba-a26b-4775-b8b0-9bd7d09c3591 req-70ef2422-ca0a-4637-90f7-05ebebf5e4b5 service nova] Acquired lock "refresh_cache-0224e4da-3079-477f-96d8-671985e85c39" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 753.054821] env[62499]: DEBUG nova.network.neutron [req-751defba-a26b-4775-b8b0-9bd7d09c3591 req-70ef2422-ca0a-4637-90f7-05ebebf5e4b5 service nova] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Refreshing network info cache for port 0a9d41a4-fdd1-4ccc-82a7-302751c3e083 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 753.219191] env[62499]: DEBUG nova.scheduler.client.report [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 753.226681] env[62499]: ERROR nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 0a9d41a4-fdd1-4ccc-82a7-302751c3e083, please check neutron logs for more information. [ 753.226681] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 753.226681] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 753.226681] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 753.226681] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 753.226681] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 753.226681] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 753.226681] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 753.226681] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 753.226681] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 753.226681] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 753.226681] env[62499]: ERROR nova.compute.manager raise self.value [ 753.226681] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 753.226681] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 753.226681] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 753.226681] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 753.227508] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 753.227508] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 753.227508] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 0a9d41a4-fdd1-4ccc-82a7-302751c3e083, please check neutron logs for more information. [ 753.227508] env[62499]: ERROR nova.compute.manager [ 753.227508] env[62499]: Traceback (most recent call last): [ 753.227508] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 753.227508] env[62499]: listener.cb(fileno) [ 753.227508] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 753.227508] env[62499]: result = function(*args, **kwargs) [ 753.227508] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 753.227508] env[62499]: return func(*args, **kwargs) [ 753.227508] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 753.227508] env[62499]: raise e [ 753.227508] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 753.227508] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 753.227508] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 753.227508] env[62499]: created_port_ids = self._update_ports_for_instance( [ 753.227508] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 753.227508] env[62499]: with excutils.save_and_reraise_exception(): [ 753.227508] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 753.227508] env[62499]: self.force_reraise() [ 753.227508] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 753.227508] env[62499]: raise self.value [ 753.227508] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 753.227508] env[62499]: updated_port = self._update_port( [ 753.227508] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 753.227508] env[62499]: _ensure_no_port_binding_failure(port) [ 753.227508] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 753.227508] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 753.229270] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 0a9d41a4-fdd1-4ccc-82a7-302751c3e083, please check neutron logs for more information. [ 753.229270] env[62499]: Removing descriptor: 16 [ 753.275986] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 753.301956] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 753.302204] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 753.302354] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 753.302534] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 753.302674] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 753.302844] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 753.303066] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 753.303220] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 753.303378] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 753.303532] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 753.303691] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 753.304826] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dbfc500d-aaa1-468e-9100-32575842d1e5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.312796] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c947bab1-cf20-434e-b0e4-98cc4b7f58a0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.327564] env[62499]: ERROR nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 0a9d41a4-fdd1-4ccc-82a7-302751c3e083, please check neutron logs for more information. [ 753.327564] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] Traceback (most recent call last): [ 753.327564] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 753.327564] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] yield resources [ 753.327564] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 753.327564] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] self.driver.spawn(context, instance, image_meta, [ 753.327564] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 753.327564] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] self._vmops.spawn(context, instance, image_meta, injected_files, [ 753.327564] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 753.327564] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] vm_ref = self.build_virtual_machine(instance, [ 753.327564] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 753.327988] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] vif_infos = vmwarevif.get_vif_info(self._session, [ 753.327988] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 753.327988] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] for vif in network_info: [ 753.327988] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 753.327988] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] return self._sync_wrapper(fn, *args, **kwargs) [ 753.327988] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 753.327988] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] self.wait() [ 753.327988] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 753.327988] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] self[:] = self._gt.wait() [ 753.327988] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 753.327988] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] return self._exit_event.wait() [ 753.327988] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 753.327988] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] current.throw(*self._exc) [ 753.328438] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 753.328438] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] result = function(*args, **kwargs) [ 753.328438] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 753.328438] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] return func(*args, **kwargs) [ 753.328438] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 753.328438] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] raise e [ 753.328438] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 753.328438] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] nwinfo = self.network_api.allocate_for_instance( [ 753.328438] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 753.328438] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] created_port_ids = self._update_ports_for_instance( [ 753.328438] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 753.328438] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] with excutils.save_and_reraise_exception(): [ 753.328438] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 753.328875] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] self.force_reraise() [ 753.328875] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 753.328875] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] raise self.value [ 753.328875] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 753.328875] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] updated_port = self._update_port( [ 753.328875] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 753.328875] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] _ensure_no_port_binding_failure(port) [ 753.328875] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 753.328875] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] raise exception.PortBindingFailed(port_id=port['id']) [ 753.328875] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] nova.exception.PortBindingFailed: Binding failed for port 0a9d41a4-fdd1-4ccc-82a7-302751c3e083, please check neutron logs for more information. [ 753.328875] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] [ 753.328875] env[62499]: INFO nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Terminating instance [ 753.330235] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquiring lock "refresh_cache-0224e4da-3079-477f-96d8-671985e85c39" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 753.475977] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 753.575036] env[62499]: DEBUG nova.network.neutron [req-751defba-a26b-4775-b8b0-9bd7d09c3591 req-70ef2422-ca0a-4637-90f7-05ebebf5e4b5 service nova] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 753.654956] env[62499]: DEBUG nova.network.neutron [req-751defba-a26b-4775-b8b0-9bd7d09c3591 req-70ef2422-ca0a-4637-90f7-05ebebf5e4b5 service nova] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 753.724521] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.477s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 753.727660] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 753.730695] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.537s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 754.157808] env[62499]: DEBUG oslo_concurrency.lockutils [req-751defba-a26b-4775-b8b0-9bd7d09c3591 req-70ef2422-ca0a-4637-90f7-05ebebf5e4b5 service nova] Releasing lock "refresh_cache-0224e4da-3079-477f-96d8-671985e85c39" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 754.158204] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquired lock "refresh_cache-0224e4da-3079-477f-96d8-671985e85c39" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 754.158735] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 754.234708] env[62499]: DEBUG nova.compute.utils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 754.240636] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 754.240636] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 754.291427] env[62499]: DEBUG nova.policy [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2aae85566cb84e41881222ddaa137bd5', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'dff7a8db03474072848c122886f622d7', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 754.675428] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3719791-adc8-43ce-b3ba-98536d3c869a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 754.677899] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Successfully created port: 87103792-bec8-416e-89d2-1438ea63251a {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 754.681648] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 754.687085] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2aaaf19d-693e-4839-a830-5d964290e0f3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 754.716469] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21f655c5-e0e2-44b9-a3ea-53729d59c93b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 754.726018] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-88670121-7b66-4749-8ee1-4f64e8355bd6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 754.738196] env[62499]: DEBUG nova.compute.provider_tree [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 754.740384] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 754.774959] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 755.091532] env[62499]: DEBUG nova.compute.manager [req-b7f05f91-176e-41c5-8bbe-8a36bc147c13 req-3993b5cb-1750-4233-b1e2-34743af3769c service nova] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Received event network-vif-deleted-0a9d41a4-fdd1-4ccc-82a7-302751c3e083 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 755.240971] env[62499]: DEBUG nova.scheduler.client.report [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 755.277556] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Releasing lock "refresh_cache-0224e4da-3079-477f-96d8-671985e85c39" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 755.278021] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 755.278275] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 755.278507] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-610d1ad5-39eb-4c46-a960-eaec6758a13f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 755.288754] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef8eaa75-5d1e-49c7-a0f8-02622b638483 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 755.310321] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 0224e4da-3079-477f-96d8-671985e85c39 could not be found. [ 755.310539] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 755.310779] env[62499]: INFO nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Took 0.03 seconds to destroy the instance on the hypervisor. [ 755.310938] env[62499]: DEBUG oslo.service.loopingcall [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 755.311956] env[62499]: DEBUG nova.compute.manager [-] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 755.311956] env[62499]: DEBUG nova.network.neutron [-] [instance: 0224e4da-3079-477f-96d8-671985e85c39] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 755.328553] env[62499]: DEBUG nova.network.neutron [-] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 755.639173] env[62499]: ERROR nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 87103792-bec8-416e-89d2-1438ea63251a, please check neutron logs for more information. [ 755.639173] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 755.639173] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 755.639173] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 755.639173] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 755.639173] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 755.639173] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 755.639173] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 755.639173] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 755.639173] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 755.639173] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 755.639173] env[62499]: ERROR nova.compute.manager raise self.value [ 755.639173] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 755.639173] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 755.639173] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 755.639173] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 755.639755] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 755.639755] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 755.639755] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 87103792-bec8-416e-89d2-1438ea63251a, please check neutron logs for more information. [ 755.639755] env[62499]: ERROR nova.compute.manager [ 755.639755] env[62499]: Traceback (most recent call last): [ 755.639755] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 755.639755] env[62499]: listener.cb(fileno) [ 755.639755] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 755.639755] env[62499]: result = function(*args, **kwargs) [ 755.639755] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 755.639755] env[62499]: return func(*args, **kwargs) [ 755.639755] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 755.639755] env[62499]: raise e [ 755.639755] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 755.639755] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 755.639755] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 755.639755] env[62499]: created_port_ids = self._update_ports_for_instance( [ 755.639755] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 755.639755] env[62499]: with excutils.save_and_reraise_exception(): [ 755.639755] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 755.639755] env[62499]: self.force_reraise() [ 755.639755] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 755.639755] env[62499]: raise self.value [ 755.639755] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 755.639755] env[62499]: updated_port = self._update_port( [ 755.639755] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 755.639755] env[62499]: _ensure_no_port_binding_failure(port) [ 755.639755] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 755.639755] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 755.640785] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 87103792-bec8-416e-89d2-1438ea63251a, please check neutron logs for more information. [ 755.640785] env[62499]: Removing descriptor: 16 [ 755.746511] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.016s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 755.747208] env[62499]: ERROR nova.compute.manager [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b5066e31-5b07-4469-91f7-8ef4b5ef88da, please check neutron logs for more information. [ 755.747208] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Traceback (most recent call last): [ 755.747208] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 755.747208] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] self.driver.spawn(context, instance, image_meta, [ 755.747208] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 755.747208] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 755.747208] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 755.747208] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] vm_ref = self.build_virtual_machine(instance, [ 755.747208] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 755.747208] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] vif_infos = vmwarevif.get_vif_info(self._session, [ 755.747208] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 755.747601] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] for vif in network_info: [ 755.747601] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 755.747601] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] return self._sync_wrapper(fn, *args, **kwargs) [ 755.747601] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 755.747601] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] self.wait() [ 755.747601] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 755.747601] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] self[:] = self._gt.wait() [ 755.747601] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 755.747601] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] return self._exit_event.wait() [ 755.747601] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 755.747601] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] result = hub.switch() [ 755.747601] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 755.747601] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] return self.greenlet.switch() [ 755.748159] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 755.748159] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] result = function(*args, **kwargs) [ 755.748159] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 755.748159] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] return func(*args, **kwargs) [ 755.748159] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 755.748159] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] raise e [ 755.748159] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 755.748159] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] nwinfo = self.network_api.allocate_for_instance( [ 755.748159] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 755.748159] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] created_port_ids = self._update_ports_for_instance( [ 755.748159] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 755.748159] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] with excutils.save_and_reraise_exception(): [ 755.748159] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 755.748590] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] self.force_reraise() [ 755.748590] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 755.748590] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] raise self.value [ 755.748590] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 755.748590] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] updated_port = self._update_port( [ 755.748590] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 755.748590] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] _ensure_no_port_binding_failure(port) [ 755.748590] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 755.748590] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] raise exception.PortBindingFailed(port_id=port['id']) [ 755.748590] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] nova.exception.PortBindingFailed: Binding failed for port b5066e31-5b07-4469-91f7-8ef4b5ef88da, please check neutron logs for more information. [ 755.748590] env[62499]: ERROR nova.compute.manager [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] [ 755.748961] env[62499]: DEBUG nova.compute.utils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Binding failed for port b5066e31-5b07-4469-91f7-8ef4b5ef88da, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 755.749311] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.525s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 755.751334] env[62499]: INFO nova.compute.claims [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 755.755988] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 755.759755] env[62499]: DEBUG nova.compute.manager [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Build of instance 75b04055-8c7e-4659-9451-ddcd6d39fe2e was re-scheduled: Binding failed for port b5066e31-5b07-4469-91f7-8ef4b5ef88da, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 755.762428] env[62499]: DEBUG nova.compute.manager [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 755.762790] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "refresh_cache-75b04055-8c7e-4659-9451-ddcd6d39fe2e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 755.763078] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquired lock "refresh_cache-75b04055-8c7e-4659-9451-ddcd6d39fe2e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 755.763373] env[62499]: DEBUG nova.network.neutron [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 755.797766] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 755.798193] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 755.798456] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 755.798799] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 755.799081] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 755.799375] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 755.799778] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 755.800113] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 755.800402] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 755.800727] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 755.801031] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 755.802778] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c718f2c-7c7e-4349-8969-8bd582d79ccd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 755.815181] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ccb792c-27c2-47c0-a302-183a7c68c561 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 755.835849] env[62499]: DEBUG nova.network.neutron [-] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 755.838365] env[62499]: ERROR nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 87103792-bec8-416e-89d2-1438ea63251a, please check neutron logs for more information. [ 755.838365] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Traceback (most recent call last): [ 755.838365] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 755.838365] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] yield resources [ 755.838365] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 755.838365] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] self.driver.spawn(context, instance, image_meta, [ 755.838365] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 755.838365] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] self._vmops.spawn(context, instance, image_meta, injected_files, [ 755.838365] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 755.838365] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] vm_ref = self.build_virtual_machine(instance, [ 755.838365] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 755.839159] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] vif_infos = vmwarevif.get_vif_info(self._session, [ 755.839159] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 755.839159] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] for vif in network_info: [ 755.839159] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 755.839159] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] return self._sync_wrapper(fn, *args, **kwargs) [ 755.839159] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 755.839159] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] self.wait() [ 755.839159] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 755.839159] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] self[:] = self._gt.wait() [ 755.839159] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 755.839159] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] return self._exit_event.wait() [ 755.839159] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 755.839159] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] current.throw(*self._exc) [ 755.839719] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 755.839719] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] result = function(*args, **kwargs) [ 755.839719] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 755.839719] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] return func(*args, **kwargs) [ 755.839719] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 755.839719] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] raise e [ 755.839719] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 755.839719] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] nwinfo = self.network_api.allocate_for_instance( [ 755.839719] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 755.839719] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] created_port_ids = self._update_ports_for_instance( [ 755.839719] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 755.839719] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] with excutils.save_and_reraise_exception(): [ 755.839719] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 755.840924] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] self.force_reraise() [ 755.840924] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 755.840924] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] raise self.value [ 755.840924] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 755.840924] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] updated_port = self._update_port( [ 755.840924] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 755.840924] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] _ensure_no_port_binding_failure(port) [ 755.840924] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 755.840924] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] raise exception.PortBindingFailed(port_id=port['id']) [ 755.840924] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] nova.exception.PortBindingFailed: Binding failed for port 87103792-bec8-416e-89d2-1438ea63251a, please check neutron logs for more information. [ 755.840924] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] [ 755.840924] env[62499]: INFO nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Terminating instance [ 755.841511] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquiring lock "refresh_cache-477a1892-0316-42ff-8fc8-269239cb2198" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 755.841754] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquired lock "refresh_cache-477a1892-0316-42ff-8fc8-269239cb2198" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 755.842031] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 756.283843] env[62499]: DEBUG nova.network.neutron [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 756.342305] env[62499]: INFO nova.compute.manager [-] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Took 1.03 seconds to deallocate network for instance. [ 756.346214] env[62499]: DEBUG nova.compute.claims [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 756.346495] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 756.361616] env[62499]: DEBUG nova.network.neutron [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 756.370538] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 756.448505] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 756.863555] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Releasing lock "refresh_cache-75b04055-8c7e-4659-9451-ddcd6d39fe2e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 756.863793] env[62499]: DEBUG nova.compute.manager [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 756.863974] env[62499]: DEBUG nova.compute.manager [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 756.864176] env[62499]: DEBUG nova.network.neutron [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 756.897849] env[62499]: DEBUG nova.network.neutron [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 756.950796] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Releasing lock "refresh_cache-477a1892-0316-42ff-8fc8-269239cb2198" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 756.951222] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 756.951416] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 756.952054] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c6207f9b-9ff8-4f47-8356-52a74039adac {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.960496] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d9931a5-d576-43ce-860b-52103d22a703 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.983289] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 477a1892-0316-42ff-8fc8-269239cb2198 could not be found. [ 756.983531] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 756.983719] env[62499]: INFO nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Took 0.03 seconds to destroy the instance on the hypervisor. [ 756.983954] env[62499]: DEBUG oslo.service.loopingcall [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 756.986096] env[62499]: DEBUG nova.compute.manager [-] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 756.986206] env[62499]: DEBUG nova.network.neutron [-] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 757.004492] env[62499]: DEBUG nova.network.neutron [-] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 757.117203] env[62499]: DEBUG nova.compute.manager [req-75645b2d-eaff-4ac8-8d16-308111e0dddd req-6582f3db-43e7-47f8-87ba-6e9a83bb65af service nova] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Received event network-changed-87103792-bec8-416e-89d2-1438ea63251a {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 757.117347] env[62499]: DEBUG nova.compute.manager [req-75645b2d-eaff-4ac8-8d16-308111e0dddd req-6582f3db-43e7-47f8-87ba-6e9a83bb65af service nova] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Refreshing instance network info cache due to event network-changed-87103792-bec8-416e-89d2-1438ea63251a. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 757.117597] env[62499]: DEBUG oslo_concurrency.lockutils [req-75645b2d-eaff-4ac8-8d16-308111e0dddd req-6582f3db-43e7-47f8-87ba-6e9a83bb65af service nova] Acquiring lock "refresh_cache-477a1892-0316-42ff-8fc8-269239cb2198" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 757.117745] env[62499]: DEBUG oslo_concurrency.lockutils [req-75645b2d-eaff-4ac8-8d16-308111e0dddd req-6582f3db-43e7-47f8-87ba-6e9a83bb65af service nova] Acquired lock "refresh_cache-477a1892-0316-42ff-8fc8-269239cb2198" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 757.117907] env[62499]: DEBUG nova.network.neutron [req-75645b2d-eaff-4ac8-8d16-308111e0dddd req-6582f3db-43e7-47f8-87ba-6e9a83bb65af service nova] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Refreshing network info cache for port 87103792-bec8-416e-89d2-1438ea63251a {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 757.163238] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83b2b7c7-3823-4dd5-b493-6146c648f2d4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.170625] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb72da51-0553-4b3f-ab23-855ee95c4524 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.202214] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95ec7dff-d947-4606-9091-58ff0a4d58c0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.209992] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b9151953-13eb-4a1b-8b41-4b87e962890f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.222421] env[62499]: DEBUG nova.compute.provider_tree [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 757.400206] env[62499]: DEBUG nova.network.neutron [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 757.507627] env[62499]: DEBUG nova.network.neutron [-] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 757.635902] env[62499]: DEBUG nova.network.neutron [req-75645b2d-eaff-4ac8-8d16-308111e0dddd req-6582f3db-43e7-47f8-87ba-6e9a83bb65af service nova] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 757.711630] env[62499]: DEBUG nova.network.neutron [req-75645b2d-eaff-4ac8-8d16-308111e0dddd req-6582f3db-43e7-47f8-87ba-6e9a83bb65af service nova] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 757.727193] env[62499]: DEBUG nova.scheduler.client.report [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 757.902845] env[62499]: INFO nova.compute.manager [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 75b04055-8c7e-4659-9451-ddcd6d39fe2e] Took 1.04 seconds to deallocate network for instance. [ 758.009159] env[62499]: INFO nova.compute.manager [-] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Took 1.02 seconds to deallocate network for instance. [ 758.011527] env[62499]: DEBUG nova.compute.claims [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 758.011631] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 758.214592] env[62499]: DEBUG oslo_concurrency.lockutils [req-75645b2d-eaff-4ac8-8d16-308111e0dddd req-6582f3db-43e7-47f8-87ba-6e9a83bb65af service nova] Releasing lock "refresh_cache-477a1892-0316-42ff-8fc8-269239cb2198" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 758.214874] env[62499]: DEBUG nova.compute.manager [req-75645b2d-eaff-4ac8-8d16-308111e0dddd req-6582f3db-43e7-47f8-87ba-6e9a83bb65af service nova] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Received event network-vif-deleted-87103792-bec8-416e-89d2-1438ea63251a {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 758.231950] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.483s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 758.232540] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 758.234974] env[62499]: DEBUG oslo_concurrency.lockutils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.591s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 758.739589] env[62499]: DEBUG nova.compute.utils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 758.744907] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 758.744907] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 758.784639] env[62499]: DEBUG nova.policy [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2aae85566cb84e41881222ddaa137bd5', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'dff7a8db03474072848c122886f622d7', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 758.938626] env[62499]: INFO nova.scheduler.client.report [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Deleted allocations for instance 75b04055-8c7e-4659-9451-ddcd6d39fe2e [ 759.081698] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Successfully created port: 7aef66d5-4e71-41da-a7f8-5ea10eb75abd {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 759.101820] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81e9e19f-a5e1-4581-8d53-c4551b67d7d1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 759.109825] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c266c4ff-a254-4171-ae15-4d7724b2a40e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 759.139276] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-796fe960-775a-4bf8-9776-979b6eba1d2f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 759.146933] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-abbfe37e-6c1e-419c-8505-39df528d7927 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 759.160780] env[62499]: DEBUG nova.compute.provider_tree [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 759.246022] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 759.448364] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c035c9c6-f75d-4eea-a7ca-6cd5a5fae1b7 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "75b04055-8c7e-4659-9451-ddcd6d39fe2e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 136.608s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 759.665058] env[62499]: DEBUG nova.scheduler.client.report [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 759.955414] env[62499]: DEBUG nova.compute.manager [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 759.987293] env[62499]: DEBUG nova.compute.manager [req-c701fdc4-e559-4d52-8738-8b6556e3230d req-02924b23-24fb-45ee-9680-08d064fafc25 service nova] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Received event network-changed-7aef66d5-4e71-41da-a7f8-5ea10eb75abd {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 759.987293] env[62499]: DEBUG nova.compute.manager [req-c701fdc4-e559-4d52-8738-8b6556e3230d req-02924b23-24fb-45ee-9680-08d064fafc25 service nova] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Refreshing instance network info cache due to event network-changed-7aef66d5-4e71-41da-a7f8-5ea10eb75abd. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 759.987293] env[62499]: DEBUG oslo_concurrency.lockutils [req-c701fdc4-e559-4d52-8738-8b6556e3230d req-02924b23-24fb-45ee-9680-08d064fafc25 service nova] Acquiring lock "refresh_cache-399b809e-8926-4348-bfaf-3499de57c1b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 759.987293] env[62499]: DEBUG oslo_concurrency.lockutils [req-c701fdc4-e559-4d52-8738-8b6556e3230d req-02924b23-24fb-45ee-9680-08d064fafc25 service nova] Acquired lock "refresh_cache-399b809e-8926-4348-bfaf-3499de57c1b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 759.987293] env[62499]: DEBUG nova.network.neutron [req-c701fdc4-e559-4d52-8738-8b6556e3230d req-02924b23-24fb-45ee-9680-08d064fafc25 service nova] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Refreshing network info cache for port 7aef66d5-4e71-41da-a7f8-5ea10eb75abd {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 760.118353] env[62499]: ERROR nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 7aef66d5-4e71-41da-a7f8-5ea10eb75abd, please check neutron logs for more information. [ 760.118353] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 760.118353] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 760.118353] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 760.118353] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 760.118353] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 760.118353] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 760.118353] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 760.118353] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 760.118353] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 760.118353] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 760.118353] env[62499]: ERROR nova.compute.manager raise self.value [ 760.118353] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 760.118353] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 760.118353] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 760.118353] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 760.119233] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 760.119233] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 760.119233] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 7aef66d5-4e71-41da-a7f8-5ea10eb75abd, please check neutron logs for more information. [ 760.119233] env[62499]: ERROR nova.compute.manager [ 760.119233] env[62499]: Traceback (most recent call last): [ 760.119233] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 760.119233] env[62499]: listener.cb(fileno) [ 760.119233] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 760.119233] env[62499]: result = function(*args, **kwargs) [ 760.119233] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 760.119233] env[62499]: return func(*args, **kwargs) [ 760.119233] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 760.119233] env[62499]: raise e [ 760.119233] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 760.119233] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 760.119233] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 760.119233] env[62499]: created_port_ids = self._update_ports_for_instance( [ 760.119233] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 760.119233] env[62499]: with excutils.save_and_reraise_exception(): [ 760.119233] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 760.119233] env[62499]: self.force_reraise() [ 760.119233] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 760.119233] env[62499]: raise self.value [ 760.119233] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 760.119233] env[62499]: updated_port = self._update_port( [ 760.119233] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 760.119233] env[62499]: _ensure_no_port_binding_failure(port) [ 760.119233] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 760.119233] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 760.120586] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 7aef66d5-4e71-41da-a7f8-5ea10eb75abd, please check neutron logs for more information. [ 760.120586] env[62499]: Removing descriptor: 17 [ 760.171607] env[62499]: DEBUG oslo_concurrency.lockutils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.937s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 760.172274] env[62499]: ERROR nova.compute.manager [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ab3d96c4-7e66-4d1c-9ee7-2acc85624f4a, please check neutron logs for more information. [ 760.172274] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] Traceback (most recent call last): [ 760.172274] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 760.172274] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] self.driver.spawn(context, instance, image_meta, [ 760.172274] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 760.172274] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] self._vmops.spawn(context, instance, image_meta, injected_files, [ 760.172274] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 760.172274] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] vm_ref = self.build_virtual_machine(instance, [ 760.172274] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 760.172274] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] vif_infos = vmwarevif.get_vif_info(self._session, [ 760.172274] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 760.172714] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] for vif in network_info: [ 760.172714] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 760.172714] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] return self._sync_wrapper(fn, *args, **kwargs) [ 760.172714] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 760.172714] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] self.wait() [ 760.172714] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 760.172714] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] self[:] = self._gt.wait() [ 760.172714] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 760.172714] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] return self._exit_event.wait() [ 760.172714] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 760.172714] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] result = hub.switch() [ 760.172714] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 760.172714] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] return self.greenlet.switch() [ 760.173224] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 760.173224] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] result = function(*args, **kwargs) [ 760.173224] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 760.173224] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] return func(*args, **kwargs) [ 760.173224] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 760.173224] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] raise e [ 760.173224] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 760.173224] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] nwinfo = self.network_api.allocate_for_instance( [ 760.173224] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 760.173224] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] created_port_ids = self._update_ports_for_instance( [ 760.173224] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 760.173224] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] with excutils.save_and_reraise_exception(): [ 760.173224] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 760.173650] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] self.force_reraise() [ 760.173650] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 760.173650] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] raise self.value [ 760.173650] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 760.173650] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] updated_port = self._update_port( [ 760.173650] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 760.173650] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] _ensure_no_port_binding_failure(port) [ 760.173650] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 760.173650] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] raise exception.PortBindingFailed(port_id=port['id']) [ 760.173650] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] nova.exception.PortBindingFailed: Binding failed for port ab3d96c4-7e66-4d1c-9ee7-2acc85624f4a, please check neutron logs for more information. [ 760.173650] env[62499]: ERROR nova.compute.manager [instance: 78a83324-36da-457e-a78d-c82d0a722015] [ 760.174208] env[62499]: DEBUG nova.compute.utils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Binding failed for port ab3d96c4-7e66-4d1c-9ee7-2acc85624f4a, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 760.174208] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.414s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 760.175643] env[62499]: INFO nova.compute.claims [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 760.180244] env[62499]: DEBUG nova.compute.manager [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Build of instance 78a83324-36da-457e-a78d-c82d0a722015 was re-scheduled: Binding failed for port ab3d96c4-7e66-4d1c-9ee7-2acc85624f4a, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 760.180244] env[62499]: DEBUG nova.compute.manager [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 760.180397] env[62499]: DEBUG oslo_concurrency.lockutils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Acquiring lock "refresh_cache-78a83324-36da-457e-a78d-c82d0a722015" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 760.180541] env[62499]: DEBUG oslo_concurrency.lockutils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Acquired lock "refresh_cache-78a83324-36da-457e-a78d-c82d0a722015" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 760.180765] env[62499]: DEBUG nova.network.neutron [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 760.253335] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 760.277186] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 760.277446] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 760.277602] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 760.277782] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 760.277926] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 760.278094] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 760.278410] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 760.278619] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 760.278801] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 760.278963] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 760.279149] env[62499]: DEBUG nova.virt.hardware [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 760.280330] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-31c9e3a1-e388-48e1-8e7a-f9404aba0105 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 760.288302] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84b6da82-2865-4108-bd52-3a6d6e87c58e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 760.301578] env[62499]: ERROR nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 7aef66d5-4e71-41da-a7f8-5ea10eb75abd, please check neutron logs for more information. [ 760.301578] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Traceback (most recent call last): [ 760.301578] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 760.301578] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] yield resources [ 760.301578] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 760.301578] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] self.driver.spawn(context, instance, image_meta, [ 760.301578] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 760.301578] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 760.301578] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 760.301578] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] vm_ref = self.build_virtual_machine(instance, [ 760.301578] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 760.302148] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] vif_infos = vmwarevif.get_vif_info(self._session, [ 760.302148] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 760.302148] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] for vif in network_info: [ 760.302148] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 760.302148] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] return self._sync_wrapper(fn, *args, **kwargs) [ 760.302148] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 760.302148] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] self.wait() [ 760.302148] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 760.302148] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] self[:] = self._gt.wait() [ 760.302148] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 760.302148] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] return self._exit_event.wait() [ 760.302148] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 760.302148] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] current.throw(*self._exc) [ 760.302579] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 760.302579] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] result = function(*args, **kwargs) [ 760.302579] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 760.302579] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] return func(*args, **kwargs) [ 760.302579] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 760.302579] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] raise e [ 760.302579] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 760.302579] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] nwinfo = self.network_api.allocate_for_instance( [ 760.302579] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 760.302579] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] created_port_ids = self._update_ports_for_instance( [ 760.302579] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 760.302579] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] with excutils.save_and_reraise_exception(): [ 760.302579] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 760.303023] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] self.force_reraise() [ 760.303023] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 760.303023] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] raise self.value [ 760.303023] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 760.303023] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] updated_port = self._update_port( [ 760.303023] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 760.303023] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] _ensure_no_port_binding_failure(port) [ 760.303023] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 760.303023] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] raise exception.PortBindingFailed(port_id=port['id']) [ 760.303023] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] nova.exception.PortBindingFailed: Binding failed for port 7aef66d5-4e71-41da-a7f8-5ea10eb75abd, please check neutron logs for more information. [ 760.303023] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] [ 760.303023] env[62499]: INFO nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Terminating instance [ 760.304014] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquiring lock "refresh_cache-399b809e-8926-4348-bfaf-3499de57c1b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 760.474658] env[62499]: DEBUG oslo_concurrency.lockutils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 760.504316] env[62499]: DEBUG nova.network.neutron [req-c701fdc4-e559-4d52-8738-8b6556e3230d req-02924b23-24fb-45ee-9680-08d064fafc25 service nova] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 760.564491] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "764ef9ab-22ae-45cd-8b5d-76a63346d93c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 760.565092] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "764ef9ab-22ae-45cd-8b5d-76a63346d93c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 760.589347] env[62499]: DEBUG nova.network.neutron [req-c701fdc4-e559-4d52-8738-8b6556e3230d req-02924b23-24fb-45ee-9680-08d064fafc25 service nova] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 760.702557] env[62499]: DEBUG nova.network.neutron [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 760.786255] env[62499]: DEBUG nova.network.neutron [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 761.091944] env[62499]: DEBUG oslo_concurrency.lockutils [req-c701fdc4-e559-4d52-8738-8b6556e3230d req-02924b23-24fb-45ee-9680-08d064fafc25 service nova] Releasing lock "refresh_cache-399b809e-8926-4348-bfaf-3499de57c1b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 761.092413] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquired lock "refresh_cache-399b809e-8926-4348-bfaf-3499de57c1b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 761.092598] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 761.291200] env[62499]: DEBUG oslo_concurrency.lockutils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Releasing lock "refresh_cache-78a83324-36da-457e-a78d-c82d0a722015" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 761.291451] env[62499]: DEBUG nova.compute.manager [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 761.291633] env[62499]: DEBUG nova.compute.manager [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 761.291797] env[62499]: DEBUG nova.network.neutron [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 761.306728] env[62499]: DEBUG nova.network.neutron [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 761.530320] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8c54283-b00f-415f-8955-0a2654074b7f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 761.538065] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4494d13-3910-4951-905d-1987e34489ee {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 761.566541] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c542297-55b0-4c36-bf31-537e08376bab {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 761.573624] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-05cc1cac-b2dd-45ff-922e-232f669f1f15 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 761.586523] env[62499]: DEBUG nova.compute.provider_tree [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 761.612290] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 761.702706] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 761.809399] env[62499]: DEBUG nova.network.neutron [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 762.013816] env[62499]: DEBUG nova.compute.manager [req-5fd298cc-21c1-4475-aa28-1a17e2d8d45e req-acdde2a3-d41a-4bf9-8ef4-88e3f118fbf2 service nova] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Received event network-vif-deleted-7aef66d5-4e71-41da-a7f8-5ea10eb75abd {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 762.090147] env[62499]: DEBUG nova.scheduler.client.report [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 762.206022] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Releasing lock "refresh_cache-399b809e-8926-4348-bfaf-3499de57c1b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 762.206357] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 762.206357] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 762.206532] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ad3d8198-9910-409a-9b6a-c1a3458399ad {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.215375] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4bc6d7a0-98a7-4006-b8bb-f38b7307f4e6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.236132] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 399b809e-8926-4348-bfaf-3499de57c1b3 could not be found. [ 762.236418] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 762.236628] env[62499]: INFO nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Took 0.03 seconds to destroy the instance on the hypervisor. [ 762.236868] env[62499]: DEBUG oslo.service.loopingcall [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 762.237405] env[62499]: DEBUG nova.compute.manager [-] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 762.237501] env[62499]: DEBUG nova.network.neutron [-] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 762.256916] env[62499]: DEBUG nova.network.neutron [-] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 762.312270] env[62499]: INFO nova.compute.manager [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] [instance: 78a83324-36da-457e-a78d-c82d0a722015] Took 1.02 seconds to deallocate network for instance. [ 762.594966] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.421s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 762.595530] env[62499]: DEBUG nova.compute.manager [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 762.598169] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 18.829s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 762.598340] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 762.598488] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62499) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 762.598762] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.258s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 762.601892] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-208d3ef3-00f9-4d07-80fc-feec996da615 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.610316] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a1f83a9-26ac-4442-bf71-4e47bab0a4fc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.624223] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ce295a5-baf0-4a71-944c-6a6d036c5240 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.631075] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e3f496f-8221-447a-befb-0652a315e967 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.661807] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181494MB free_disk=130GB free_vcpus=48 pci_devices=None {{(pid=62499) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 762.661965] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 762.760040] env[62499]: DEBUG nova.network.neutron [-] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 763.103255] env[62499]: DEBUG nova.compute.utils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 763.104964] env[62499]: DEBUG nova.compute.manager [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 763.105159] env[62499]: DEBUG nova.network.neutron [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 763.146840] env[62499]: DEBUG nova.policy [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '88c76f25140f4de28bd719de1d5ff616', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'cb9ca6d8c8ae4352bc3c7b1daa7ba288', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 763.262327] env[62499]: INFO nova.compute.manager [-] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Took 1.02 seconds to deallocate network for instance. [ 763.264809] env[62499]: DEBUG nova.compute.claims [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 763.264904] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 763.344344] env[62499]: INFO nova.scheduler.client.report [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Deleted allocations for instance 78a83324-36da-457e-a78d-c82d0a722015 [ 763.487600] env[62499]: DEBUG nova.network.neutron [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Successfully created port: 8d274454-fb43-45a5-9745-eda3d5e00015 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 763.491827] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e1173ff-df89-4d3e-8aa4-82d6413f84b8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 763.500017] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75b26465-6c03-49a2-a38c-066a413753e4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 763.530464] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6cce8d8-f244-4868-a4e7-3371deaad1b5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 763.538157] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bfdbf745-b9b1-46e1-9112-d4e3e08d2a44 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 763.552186] env[62499]: DEBUG nova.compute.provider_tree [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 763.613085] env[62499]: DEBUG nova.compute.manager [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 763.858468] env[62499]: DEBUG oslo_concurrency.lockutils [None req-424db3ec-4aa1-4116-b20a-8ebc29792225 tempest-AttachInterfacesV270Test-336752351 tempest-AttachInterfacesV270Test-336752351-project-member] Lock "78a83324-36da-457e-a78d-c82d0a722015" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 135.136s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 764.056035] env[62499]: DEBUG nova.scheduler.client.report [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 764.193667] env[62499]: DEBUG nova.compute.manager [req-b1f5cb95-5dca-4449-a82e-0c67a940a820 req-516ce6a2-0563-410e-bbe1-4f732aeeb27d service nova] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Received event network-changed-8d274454-fb43-45a5-9745-eda3d5e00015 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 764.193873] env[62499]: DEBUG nova.compute.manager [req-b1f5cb95-5dca-4449-a82e-0c67a940a820 req-516ce6a2-0563-410e-bbe1-4f732aeeb27d service nova] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Refreshing instance network info cache due to event network-changed-8d274454-fb43-45a5-9745-eda3d5e00015. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 764.194087] env[62499]: DEBUG oslo_concurrency.lockutils [req-b1f5cb95-5dca-4449-a82e-0c67a940a820 req-516ce6a2-0563-410e-bbe1-4f732aeeb27d service nova] Acquiring lock "refresh_cache-6e078697-9fc5-4605-8d4a-a47d9331ba9c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 764.194232] env[62499]: DEBUG oslo_concurrency.lockutils [req-b1f5cb95-5dca-4449-a82e-0c67a940a820 req-516ce6a2-0563-410e-bbe1-4f732aeeb27d service nova] Acquired lock "refresh_cache-6e078697-9fc5-4605-8d4a-a47d9331ba9c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 764.194390] env[62499]: DEBUG nova.network.neutron [req-b1f5cb95-5dca-4449-a82e-0c67a940a820 req-516ce6a2-0563-410e-bbe1-4f732aeeb27d service nova] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Refreshing network info cache for port 8d274454-fb43-45a5-9745-eda3d5e00015 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 764.360446] env[62499]: DEBUG nova.compute.manager [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 764.441172] env[62499]: ERROR nova.compute.manager [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 8d274454-fb43-45a5-9745-eda3d5e00015, please check neutron logs for more information. [ 764.441172] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 764.441172] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 764.441172] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 764.441172] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 764.441172] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 764.441172] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 764.441172] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 764.441172] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 764.441172] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 764.441172] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 764.441172] env[62499]: ERROR nova.compute.manager raise self.value [ 764.441172] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 764.441172] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 764.441172] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 764.441172] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 764.441749] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 764.441749] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 764.441749] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 8d274454-fb43-45a5-9745-eda3d5e00015, please check neutron logs for more information. [ 764.441749] env[62499]: ERROR nova.compute.manager [ 764.441749] env[62499]: Traceback (most recent call last): [ 764.441749] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 764.441749] env[62499]: listener.cb(fileno) [ 764.441749] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 764.441749] env[62499]: result = function(*args, **kwargs) [ 764.441749] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 764.441749] env[62499]: return func(*args, **kwargs) [ 764.441749] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 764.441749] env[62499]: raise e [ 764.441749] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 764.441749] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 764.441749] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 764.441749] env[62499]: created_port_ids = self._update_ports_for_instance( [ 764.441749] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 764.441749] env[62499]: with excutils.save_and_reraise_exception(): [ 764.441749] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 764.441749] env[62499]: self.force_reraise() [ 764.441749] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 764.441749] env[62499]: raise self.value [ 764.441749] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 764.441749] env[62499]: updated_port = self._update_port( [ 764.441749] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 764.441749] env[62499]: _ensure_no_port_binding_failure(port) [ 764.441749] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 764.441749] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 764.442710] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 8d274454-fb43-45a5-9745-eda3d5e00015, please check neutron logs for more information. [ 764.442710] env[62499]: Removing descriptor: 17 [ 764.562758] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.964s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 764.563396] env[62499]: ERROR nova.compute.manager [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 724ced36-552b-431b-84db-b53ae96f6fda, please check neutron logs for more information. [ 764.563396] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Traceback (most recent call last): [ 764.563396] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 764.563396] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] self.driver.spawn(context, instance, image_meta, [ 764.563396] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 764.563396] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] self._vmops.spawn(context, instance, image_meta, injected_files, [ 764.563396] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 764.563396] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] vm_ref = self.build_virtual_machine(instance, [ 764.563396] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 764.563396] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] vif_infos = vmwarevif.get_vif_info(self._session, [ 764.563396] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 764.563744] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] for vif in network_info: [ 764.563744] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 764.563744] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] return self._sync_wrapper(fn, *args, **kwargs) [ 764.563744] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 764.563744] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] self.wait() [ 764.563744] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 764.563744] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] self[:] = self._gt.wait() [ 764.563744] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 764.563744] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] return self._exit_event.wait() [ 764.563744] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 764.563744] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] result = hub.switch() [ 764.563744] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 764.563744] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] return self.greenlet.switch() [ 764.564111] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 764.564111] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] result = function(*args, **kwargs) [ 764.564111] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 764.564111] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] return func(*args, **kwargs) [ 764.564111] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 764.564111] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] raise e [ 764.564111] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 764.564111] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] nwinfo = self.network_api.allocate_for_instance( [ 764.564111] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 764.564111] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] created_port_ids = self._update_ports_for_instance( [ 764.564111] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 764.564111] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] with excutils.save_and_reraise_exception(): [ 764.564111] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 764.564485] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] self.force_reraise() [ 764.564485] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 764.564485] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] raise self.value [ 764.564485] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 764.564485] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] updated_port = self._update_port( [ 764.564485] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 764.564485] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] _ensure_no_port_binding_failure(port) [ 764.564485] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 764.564485] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] raise exception.PortBindingFailed(port_id=port['id']) [ 764.564485] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] nova.exception.PortBindingFailed: Binding failed for port 724ced36-552b-431b-84db-b53ae96f6fda, please check neutron logs for more information. [ 764.564485] env[62499]: ERROR nova.compute.manager [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] [ 764.564794] env[62499]: DEBUG nova.compute.utils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Binding failed for port 724ced36-552b-431b-84db-b53ae96f6fda, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 764.565451] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.907s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 764.568039] env[62499]: DEBUG nova.compute.manager [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Build of instance dee17e73-22db-48ff-b0b0-ec7bec850c48 was re-scheduled: Binding failed for port 724ced36-552b-431b-84db-b53ae96f6fda, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 764.568453] env[62499]: DEBUG nova.compute.manager [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 764.568673] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Acquiring lock "refresh_cache-dee17e73-22db-48ff-b0b0-ec7bec850c48" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 764.568816] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Acquired lock "refresh_cache-dee17e73-22db-48ff-b0b0-ec7bec850c48" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 764.568969] env[62499]: DEBUG nova.network.neutron [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 764.621284] env[62499]: DEBUG nova.compute.manager [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 764.646130] env[62499]: DEBUG nova.virt.hardware [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 764.646371] env[62499]: DEBUG nova.virt.hardware [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 764.646526] env[62499]: DEBUG nova.virt.hardware [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 764.646705] env[62499]: DEBUG nova.virt.hardware [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 764.646851] env[62499]: DEBUG nova.virt.hardware [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 764.646996] env[62499]: DEBUG nova.virt.hardware [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 764.647214] env[62499]: DEBUG nova.virt.hardware [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 764.647373] env[62499]: DEBUG nova.virt.hardware [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 764.647532] env[62499]: DEBUG nova.virt.hardware [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 764.647692] env[62499]: DEBUG nova.virt.hardware [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 764.647862] env[62499]: DEBUG nova.virt.hardware [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 764.648740] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5cf64832-c2e2-46f0-a1b5-9a5499045c21 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 764.656816] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-841d9bf5-bade-4eb4-ac3f-b99772cfabb9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 764.670996] env[62499]: ERROR nova.compute.manager [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 8d274454-fb43-45a5-9745-eda3d5e00015, please check neutron logs for more information. [ 764.670996] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Traceback (most recent call last): [ 764.670996] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 764.670996] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] yield resources [ 764.670996] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 764.670996] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] self.driver.spawn(context, instance, image_meta, [ 764.670996] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 764.670996] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 764.670996] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 764.670996] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] vm_ref = self.build_virtual_machine(instance, [ 764.670996] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 764.671367] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] vif_infos = vmwarevif.get_vif_info(self._session, [ 764.671367] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 764.671367] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] for vif in network_info: [ 764.671367] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 764.671367] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] return self._sync_wrapper(fn, *args, **kwargs) [ 764.671367] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 764.671367] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] self.wait() [ 764.671367] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 764.671367] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] self[:] = self._gt.wait() [ 764.671367] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 764.671367] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] return self._exit_event.wait() [ 764.671367] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 764.671367] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] current.throw(*self._exc) [ 764.671865] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 764.671865] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] result = function(*args, **kwargs) [ 764.671865] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 764.671865] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] return func(*args, **kwargs) [ 764.671865] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 764.671865] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] raise e [ 764.671865] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 764.671865] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] nwinfo = self.network_api.allocate_for_instance( [ 764.671865] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 764.671865] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] created_port_ids = self._update_ports_for_instance( [ 764.671865] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 764.671865] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] with excutils.save_and_reraise_exception(): [ 764.671865] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 764.673911] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] self.force_reraise() [ 764.673911] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 764.673911] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] raise self.value [ 764.673911] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 764.673911] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] updated_port = self._update_port( [ 764.673911] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 764.673911] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] _ensure_no_port_binding_failure(port) [ 764.673911] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 764.673911] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] raise exception.PortBindingFailed(port_id=port['id']) [ 764.673911] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] nova.exception.PortBindingFailed: Binding failed for port 8d274454-fb43-45a5-9745-eda3d5e00015, please check neutron logs for more information. [ 764.673911] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] [ 764.673911] env[62499]: INFO nova.compute.manager [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Terminating instance [ 764.674380] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquiring lock "refresh_cache-6e078697-9fc5-4605-8d4a-a47d9331ba9c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 764.719017] env[62499]: DEBUG nova.network.neutron [req-b1f5cb95-5dca-4449-a82e-0c67a940a820 req-516ce6a2-0563-410e-bbe1-4f732aeeb27d service nova] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 764.805126] env[62499]: DEBUG nova.network.neutron [req-b1f5cb95-5dca-4449-a82e-0c67a940a820 req-516ce6a2-0563-410e-bbe1-4f732aeeb27d service nova] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 764.883435] env[62499]: DEBUG oslo_concurrency.lockutils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 765.091594] env[62499]: DEBUG nova.network.neutron [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 765.199449] env[62499]: DEBUG nova.network.neutron [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 765.308526] env[62499]: DEBUG oslo_concurrency.lockutils [req-b1f5cb95-5dca-4449-a82e-0c67a940a820 req-516ce6a2-0563-410e-bbe1-4f732aeeb27d service nova] Releasing lock "refresh_cache-6e078697-9fc5-4605-8d4a-a47d9331ba9c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 765.308918] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquired lock "refresh_cache-6e078697-9fc5-4605-8d4a-a47d9331ba9c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 765.309175] env[62499]: DEBUG nova.network.neutron [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 765.445872] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-53b9a85a-5750-4fd6-b9b4-5d6a03f29f62 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.451643] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a480025-6463-4fc0-819a-7f9ec3953ee3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.481966] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc11de74-d7a1-4a0a-9059-36d9e76808a2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.489557] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f1861de-a08d-436d-b30e-fa274110be61 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.503302] env[62499]: DEBUG nova.compute.provider_tree [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 765.703674] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Releasing lock "refresh_cache-dee17e73-22db-48ff-b0b0-ec7bec850c48" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 765.703674] env[62499]: DEBUG nova.compute.manager [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 765.703674] env[62499]: DEBUG nova.compute.manager [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 765.703874] env[62499]: DEBUG nova.network.neutron [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 765.723845] env[62499]: DEBUG nova.network.neutron [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 765.841200] env[62499]: DEBUG nova.network.neutron [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 765.972681] env[62499]: DEBUG nova.network.neutron [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 766.007038] env[62499]: DEBUG nova.scheduler.client.report [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 766.221422] env[62499]: DEBUG nova.compute.manager [req-4883a939-e256-453c-92e3-770a9bea89a5 req-93aa65a2-d842-432c-8cc3-5f3ba74be30c service nova] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Received event network-vif-deleted-8d274454-fb43-45a5-9745-eda3d5e00015 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 766.226474] env[62499]: DEBUG nova.network.neutron [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 766.475840] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Releasing lock "refresh_cache-6e078697-9fc5-4605-8d4a-a47d9331ba9c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 766.476329] env[62499]: DEBUG nova.compute.manager [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 766.476519] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 766.476824] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-a6294a49-3911-40a5-bcc5-6ed3aa8418ef {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.486201] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86db1021-c162-4e5c-b225-cc4323ce9370 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.509199] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6e078697-9fc5-4605-8d4a-a47d9331ba9c could not be found. [ 766.509421] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 766.509666] env[62499]: INFO nova.compute.manager [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Took 0.03 seconds to destroy the instance on the hypervisor. [ 766.509900] env[62499]: DEBUG oslo.service.loopingcall [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 766.510131] env[62499]: DEBUG nova.compute.manager [-] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 766.510222] env[62499]: DEBUG nova.network.neutron [-] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 766.512224] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.947s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 766.512750] env[62499]: ERROR nova.compute.manager [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f2efa323-2a49-4acf-84ce-6cd3af0dcd52, please check neutron logs for more information. [ 766.512750] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Traceback (most recent call last): [ 766.512750] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 766.512750] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] self.driver.spawn(context, instance, image_meta, [ 766.512750] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 766.512750] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] self._vmops.spawn(context, instance, image_meta, injected_files, [ 766.512750] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 766.512750] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] vm_ref = self.build_virtual_machine(instance, [ 766.512750] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 766.512750] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] vif_infos = vmwarevif.get_vif_info(self._session, [ 766.512750] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 766.513151] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] for vif in network_info: [ 766.513151] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 766.513151] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] return self._sync_wrapper(fn, *args, **kwargs) [ 766.513151] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 766.513151] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] self.wait() [ 766.513151] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 766.513151] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] self[:] = self._gt.wait() [ 766.513151] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 766.513151] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] return self._exit_event.wait() [ 766.513151] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 766.513151] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] result = hub.switch() [ 766.513151] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 766.513151] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] return self.greenlet.switch() [ 766.513646] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 766.513646] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] result = function(*args, **kwargs) [ 766.513646] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 766.513646] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] return func(*args, **kwargs) [ 766.513646] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 766.513646] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] raise e [ 766.513646] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 766.513646] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] nwinfo = self.network_api.allocate_for_instance( [ 766.513646] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 766.513646] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] created_port_ids = self._update_ports_for_instance( [ 766.513646] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 766.513646] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] with excutils.save_and_reraise_exception(): [ 766.513646] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 766.514082] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] self.force_reraise() [ 766.514082] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 766.514082] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] raise self.value [ 766.514082] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 766.514082] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] updated_port = self._update_port( [ 766.514082] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 766.514082] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] _ensure_no_port_binding_failure(port) [ 766.514082] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 766.514082] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] raise exception.PortBindingFailed(port_id=port['id']) [ 766.514082] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] nova.exception.PortBindingFailed: Binding failed for port f2efa323-2a49-4acf-84ce-6cd3af0dcd52, please check neutron logs for more information. [ 766.514082] env[62499]: ERROR nova.compute.manager [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] [ 766.514444] env[62499]: DEBUG nova.compute.utils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Binding failed for port f2efa323-2a49-4acf-84ce-6cd3af0dcd52, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 766.514589] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.506s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 766.518653] env[62499]: DEBUG nova.compute.manager [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Build of instance f5b10ac7-e20c-439c-bc6f-a3cf2dd26122 was re-scheduled: Binding failed for port f2efa323-2a49-4acf-84ce-6cd3af0dcd52, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 766.519133] env[62499]: DEBUG nova.compute.manager [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 766.519360] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Acquiring lock "refresh_cache-f5b10ac7-e20c-439c-bc6f-a3cf2dd26122" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 766.519504] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Acquired lock "refresh_cache-f5b10ac7-e20c-439c-bc6f-a3cf2dd26122" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 766.519732] env[62499]: DEBUG nova.network.neutron [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 766.530129] env[62499]: DEBUG nova.network.neutron [-] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 766.728903] env[62499]: INFO nova.compute.manager [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] [instance: dee17e73-22db-48ff-b0b0-ec7bec850c48] Took 1.02 seconds to deallocate network for instance. [ 767.032262] env[62499]: DEBUG nova.network.neutron [-] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 767.036958] env[62499]: DEBUG nova.network.neutron [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 767.131798] env[62499]: DEBUG nova.network.neutron [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 767.361643] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79b9efb2-d8a5-40be-bf54-20160b0481c3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.369038] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f21fa5c-2260-404e-aad8-d5124ec6b875 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.399319] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b01340ae-c188-4764-8625-f7ea0fa07f3d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.407087] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d586c57-90cb-464c-ac04-a576eedbb929 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.420351] env[62499]: DEBUG nova.compute.provider_tree [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 767.535610] env[62499]: INFO nova.compute.manager [-] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Took 1.03 seconds to deallocate network for instance. [ 767.538069] env[62499]: DEBUG nova.compute.claims [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 767.538256] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 767.634694] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Releasing lock "refresh_cache-f5b10ac7-e20c-439c-bc6f-a3cf2dd26122" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 767.634957] env[62499]: DEBUG nova.compute.manager [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 767.635154] env[62499]: DEBUG nova.compute.manager [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 767.635328] env[62499]: DEBUG nova.network.neutron [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 767.648878] env[62499]: DEBUG nova.network.neutron [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 767.761104] env[62499]: INFO nova.scheduler.client.report [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Deleted allocations for instance dee17e73-22db-48ff-b0b0-ec7bec850c48 [ 767.923883] env[62499]: DEBUG nova.scheduler.client.report [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 768.151906] env[62499]: DEBUG nova.network.neutron [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 768.268996] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ec648a66-2364-4d11-829d-c9d9197b7972 tempest-ServersTestMultiNic-232926952 tempest-ServersTestMultiNic-232926952-project-member] Lock "dee17e73-22db-48ff-b0b0-ec7bec850c48" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 138.748s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 768.428143] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.913s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 768.428755] env[62499]: ERROR nova.compute.manager [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 9cdf2850-ba3f-49bf-afc4-6b926de1325b, please check neutron logs for more information. [ 768.428755] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Traceback (most recent call last): [ 768.428755] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 768.428755] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] self.driver.spawn(context, instance, image_meta, [ 768.428755] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 768.428755] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 768.428755] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 768.428755] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] vm_ref = self.build_virtual_machine(instance, [ 768.428755] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 768.428755] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] vif_infos = vmwarevif.get_vif_info(self._session, [ 768.428755] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 768.429181] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] for vif in network_info: [ 768.429181] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 768.429181] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] return self._sync_wrapper(fn, *args, **kwargs) [ 768.429181] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 768.429181] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] self.wait() [ 768.429181] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 768.429181] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] self[:] = self._gt.wait() [ 768.429181] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 768.429181] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] return self._exit_event.wait() [ 768.429181] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 768.429181] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] current.throw(*self._exc) [ 768.429181] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 768.429181] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] result = function(*args, **kwargs) [ 768.429645] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 768.429645] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] return func(*args, **kwargs) [ 768.429645] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 768.429645] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] raise e [ 768.429645] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 768.429645] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] nwinfo = self.network_api.allocate_for_instance( [ 768.429645] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 768.429645] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] created_port_ids = self._update_ports_for_instance( [ 768.429645] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 768.429645] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] with excutils.save_and_reraise_exception(): [ 768.429645] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 768.429645] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] self.force_reraise() [ 768.429645] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 768.430079] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] raise self.value [ 768.430079] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 768.430079] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] updated_port = self._update_port( [ 768.430079] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 768.430079] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] _ensure_no_port_binding_failure(port) [ 768.430079] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 768.430079] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] raise exception.PortBindingFailed(port_id=port['id']) [ 768.430079] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] nova.exception.PortBindingFailed: Binding failed for port 9cdf2850-ba3f-49bf-afc4-6b926de1325b, please check neutron logs for more information. [ 768.430079] env[62499]: ERROR nova.compute.manager [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] [ 768.430079] env[62499]: DEBUG nova.compute.utils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Binding failed for port 9cdf2850-ba3f-49bf-afc4-6b926de1325b, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 768.430752] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.955s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 768.432172] env[62499]: INFO nova.compute.claims [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 768.434669] env[62499]: DEBUG nova.compute.manager [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Build of instance ff822880-41dc-429f-80fb-a1ddc0441ea3 was re-scheduled: Binding failed for port 9cdf2850-ba3f-49bf-afc4-6b926de1325b, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 768.435107] env[62499]: DEBUG nova.compute.manager [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 768.435331] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Acquiring lock "refresh_cache-ff822880-41dc-429f-80fb-a1ddc0441ea3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 768.435473] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Acquired lock "refresh_cache-ff822880-41dc-429f-80fb-a1ddc0441ea3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 768.435630] env[62499]: DEBUG nova.network.neutron [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 768.654039] env[62499]: INFO nova.compute.manager [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: f5b10ac7-e20c-439c-bc6f-a3cf2dd26122] Took 1.02 seconds to deallocate network for instance. [ 768.772099] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 768.965304] env[62499]: DEBUG nova.network.neutron [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 769.056055] env[62499]: DEBUG nova.network.neutron [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 769.295739] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 769.559769] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Releasing lock "refresh_cache-ff822880-41dc-429f-80fb-a1ddc0441ea3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 769.560029] env[62499]: DEBUG nova.compute.manager [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 769.560222] env[62499]: DEBUG nova.compute.manager [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 769.560387] env[62499]: DEBUG nova.network.neutron [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 769.582877] env[62499]: DEBUG nova.network.neutron [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 769.690247] env[62499]: INFO nova.scheduler.client.report [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Deleted allocations for instance f5b10ac7-e20c-439c-bc6f-a3cf2dd26122 [ 769.819043] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe0ac5db-e9da-449c-a8ce-6f18fb8146bc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.828369] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-809fa549-d554-41c3-ba86-11e0e186e641 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.866841] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da80e15d-ed96-43cf-85a5-3a6e61a0a626 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.874473] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3afd1a49-1c32-4ab9-8322-2cc4329f3d7b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.888366] env[62499]: DEBUG nova.compute.provider_tree [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 770.085656] env[62499]: DEBUG nova.network.neutron [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 770.201476] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f9faa728-de6b-47a7-88de-8c3bf7cf5cb9 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Lock "f5b10ac7-e20c-439c-bc6f-a3cf2dd26122" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 130.352s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 770.392056] env[62499]: DEBUG nova.scheduler.client.report [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 770.588797] env[62499]: INFO nova.compute.manager [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] [instance: ff822880-41dc-429f-80fb-a1ddc0441ea3] Took 1.03 seconds to deallocate network for instance. [ 770.704513] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 770.899331] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.467s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 770.899331] env[62499]: DEBUG nova.compute.manager [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 770.900897] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.554s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 771.226891] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 771.405779] env[62499]: DEBUG nova.compute.utils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 771.410644] env[62499]: DEBUG nova.compute.manager [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 771.410644] env[62499]: DEBUG nova.network.neutron [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 771.456443] env[62499]: DEBUG nova.policy [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a49582e5ed5448a19188c19f4439a61e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '4565b45612fb4f7197f883514bf7c028', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 771.616996] env[62499]: INFO nova.scheduler.client.report [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Deleted allocations for instance ff822880-41dc-429f-80fb-a1ddc0441ea3 [ 771.732780] env[62499]: DEBUG nova.network.neutron [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Successfully created port: 7b317662-2319-4c6a-8a87-72b6d90c23aa {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 771.744873] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ecb3299-f4da-464d-a00e-8e97ef86e553 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 771.753522] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2079dd8e-3875-40b6-8a47-85f8fec590f5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 771.785538] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d97f7c80-5e5b-4f0e-b275-461d774ac2fb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 771.793534] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e6c861b-ef9e-486e-a60e-eb4b539e1e2d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 771.807648] env[62499]: DEBUG nova.compute.provider_tree [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 771.910794] env[62499]: DEBUG nova.compute.manager [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 772.128153] env[62499]: DEBUG oslo_concurrency.lockutils [None req-6a118b60-2d9d-450e-8ffc-51dc75500cd0 tempest-ServersAdminTestJSON-1716551685 tempest-ServersAdminTestJSON-1716551685-project-member] Lock "ff822880-41dc-429f-80fb-a1ddc0441ea3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 131.381s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 772.313070] env[62499]: DEBUG nova.scheduler.client.report [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 772.525164] env[62499]: DEBUG nova.compute.manager [req-1aaa3cc9-3e31-4885-a347-e39222f65a01 req-da7d8a10-3371-4e5f-be63-750ba218f11d service nova] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Received event network-changed-7b317662-2319-4c6a-8a87-72b6d90c23aa {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 772.525164] env[62499]: DEBUG nova.compute.manager [req-1aaa3cc9-3e31-4885-a347-e39222f65a01 req-da7d8a10-3371-4e5f-be63-750ba218f11d service nova] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Refreshing instance network info cache due to event network-changed-7b317662-2319-4c6a-8a87-72b6d90c23aa. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 772.525164] env[62499]: DEBUG oslo_concurrency.lockutils [req-1aaa3cc9-3e31-4885-a347-e39222f65a01 req-da7d8a10-3371-4e5f-be63-750ba218f11d service nova] Acquiring lock "refresh_cache-e7023ea7-9a39-4e5b-90f6-b2499e06ee9d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 772.525164] env[62499]: DEBUG oslo_concurrency.lockutils [req-1aaa3cc9-3e31-4885-a347-e39222f65a01 req-da7d8a10-3371-4e5f-be63-750ba218f11d service nova] Acquired lock "refresh_cache-e7023ea7-9a39-4e5b-90f6-b2499e06ee9d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 772.526235] env[62499]: DEBUG nova.network.neutron [req-1aaa3cc9-3e31-4885-a347-e39222f65a01 req-da7d8a10-3371-4e5f-be63-750ba218f11d service nova] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Refreshing network info cache for port 7b317662-2319-4c6a-8a87-72b6d90c23aa {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 772.630812] env[62499]: DEBUG nova.compute.manager [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 772.712462] env[62499]: ERROR nova.compute.manager [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 7b317662-2319-4c6a-8a87-72b6d90c23aa, please check neutron logs for more information. [ 772.712462] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 772.712462] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 772.712462] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 772.712462] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 772.712462] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 772.712462] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 772.712462] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 772.712462] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 772.712462] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 772.712462] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 772.712462] env[62499]: ERROR nova.compute.manager raise self.value [ 772.712462] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 772.712462] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 772.712462] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 772.712462] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 772.713068] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 772.713068] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 772.713068] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 7b317662-2319-4c6a-8a87-72b6d90c23aa, please check neutron logs for more information. [ 772.713068] env[62499]: ERROR nova.compute.manager [ 772.713068] env[62499]: Traceback (most recent call last): [ 772.713068] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 772.713068] env[62499]: listener.cb(fileno) [ 772.713068] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 772.713068] env[62499]: result = function(*args, **kwargs) [ 772.713068] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 772.713068] env[62499]: return func(*args, **kwargs) [ 772.713068] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 772.713068] env[62499]: raise e [ 772.713068] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 772.713068] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 772.713068] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 772.713068] env[62499]: created_port_ids = self._update_ports_for_instance( [ 772.713068] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 772.713068] env[62499]: with excutils.save_and_reraise_exception(): [ 772.713068] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 772.713068] env[62499]: self.force_reraise() [ 772.713068] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 772.713068] env[62499]: raise self.value [ 772.713068] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 772.713068] env[62499]: updated_port = self._update_port( [ 772.713068] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 772.713068] env[62499]: _ensure_no_port_binding_failure(port) [ 772.713068] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 772.713068] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 772.714073] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 7b317662-2319-4c6a-8a87-72b6d90c23aa, please check neutron logs for more information. [ 772.714073] env[62499]: Removing descriptor: 17 [ 772.818204] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.917s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 772.818973] env[62499]: ERROR nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 0a9d41a4-fdd1-4ccc-82a7-302751c3e083, please check neutron logs for more information. [ 772.818973] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] Traceback (most recent call last): [ 772.818973] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 772.818973] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] self.driver.spawn(context, instance, image_meta, [ 772.818973] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 772.818973] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] self._vmops.spawn(context, instance, image_meta, injected_files, [ 772.818973] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 772.818973] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] vm_ref = self.build_virtual_machine(instance, [ 772.818973] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 772.818973] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] vif_infos = vmwarevif.get_vif_info(self._session, [ 772.818973] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 772.819329] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] for vif in network_info: [ 772.819329] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 772.819329] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] return self._sync_wrapper(fn, *args, **kwargs) [ 772.819329] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 772.819329] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] self.wait() [ 772.819329] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 772.819329] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] self[:] = self._gt.wait() [ 772.819329] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 772.819329] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] return self._exit_event.wait() [ 772.819329] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 772.819329] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] current.throw(*self._exc) [ 772.819329] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 772.819329] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] result = function(*args, **kwargs) [ 772.819817] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 772.819817] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] return func(*args, **kwargs) [ 772.819817] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 772.819817] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] raise e [ 772.819817] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 772.819817] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] nwinfo = self.network_api.allocate_for_instance( [ 772.819817] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 772.819817] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] created_port_ids = self._update_ports_for_instance( [ 772.819817] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 772.819817] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] with excutils.save_and_reraise_exception(): [ 772.819817] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 772.819817] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] self.force_reraise() [ 772.819817] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 772.820290] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] raise self.value [ 772.820290] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 772.820290] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] updated_port = self._update_port( [ 772.820290] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 772.820290] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] _ensure_no_port_binding_failure(port) [ 772.820290] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 772.820290] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] raise exception.PortBindingFailed(port_id=port['id']) [ 772.820290] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] nova.exception.PortBindingFailed: Binding failed for port 0a9d41a4-fdd1-4ccc-82a7-302751c3e083, please check neutron logs for more information. [ 772.820290] env[62499]: ERROR nova.compute.manager [instance: 0224e4da-3079-477f-96d8-671985e85c39] [ 772.820290] env[62499]: DEBUG nova.compute.utils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Binding failed for port 0a9d41a4-fdd1-4ccc-82a7-302751c3e083, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 772.821878] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.810s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 772.825464] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Build of instance 0224e4da-3079-477f-96d8-671985e85c39 was re-scheduled: Binding failed for port 0a9d41a4-fdd1-4ccc-82a7-302751c3e083, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 772.826175] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 772.826488] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquiring lock "refresh_cache-0224e4da-3079-477f-96d8-671985e85c39" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 772.827869] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquired lock "refresh_cache-0224e4da-3079-477f-96d8-671985e85c39" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 772.828631] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 772.920821] env[62499]: DEBUG nova.compute.manager [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 772.959143] env[62499]: DEBUG nova.virt.hardware [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 772.959404] env[62499]: DEBUG nova.virt.hardware [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 772.959558] env[62499]: DEBUG nova.virt.hardware [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 772.959775] env[62499]: DEBUG nova.virt.hardware [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 772.959925] env[62499]: DEBUG nova.virt.hardware [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 772.960102] env[62499]: DEBUG nova.virt.hardware [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 772.960369] env[62499]: DEBUG nova.virt.hardware [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 772.960573] env[62499]: DEBUG nova.virt.hardware [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 772.960760] env[62499]: DEBUG nova.virt.hardware [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 772.960933] env[62499]: DEBUG nova.virt.hardware [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 772.961120] env[62499]: DEBUG nova.virt.hardware [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 772.962026] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd4c5c73-fcee-43f2-acd2-5ab7bcf9f9df {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.971453] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a44bfcc3-0432-47b6-9d0e-37c4a9de99b8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.988026] env[62499]: ERROR nova.compute.manager [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 7b317662-2319-4c6a-8a87-72b6d90c23aa, please check neutron logs for more information. [ 772.988026] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Traceback (most recent call last): [ 772.988026] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 772.988026] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] yield resources [ 772.988026] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 772.988026] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] self.driver.spawn(context, instance, image_meta, [ 772.988026] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 772.988026] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 772.988026] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 772.988026] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] vm_ref = self.build_virtual_machine(instance, [ 772.988026] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 772.988525] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] vif_infos = vmwarevif.get_vif_info(self._session, [ 772.988525] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 772.988525] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] for vif in network_info: [ 772.988525] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 772.988525] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] return self._sync_wrapper(fn, *args, **kwargs) [ 772.988525] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 772.988525] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] self.wait() [ 772.988525] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 772.988525] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] self[:] = self._gt.wait() [ 772.988525] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 772.988525] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] return self._exit_event.wait() [ 772.988525] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 772.988525] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] current.throw(*self._exc) [ 772.988934] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 772.988934] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] result = function(*args, **kwargs) [ 772.988934] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 772.988934] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] return func(*args, **kwargs) [ 772.988934] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 772.988934] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] raise e [ 772.988934] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 772.988934] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] nwinfo = self.network_api.allocate_for_instance( [ 772.988934] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 772.988934] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] created_port_ids = self._update_ports_for_instance( [ 772.988934] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 772.988934] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] with excutils.save_and_reraise_exception(): [ 772.988934] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 772.989339] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] self.force_reraise() [ 772.989339] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 772.989339] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] raise self.value [ 772.989339] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 772.989339] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] updated_port = self._update_port( [ 772.989339] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 772.989339] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] _ensure_no_port_binding_failure(port) [ 772.989339] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 772.989339] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] raise exception.PortBindingFailed(port_id=port['id']) [ 772.989339] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] nova.exception.PortBindingFailed: Binding failed for port 7b317662-2319-4c6a-8a87-72b6d90c23aa, please check neutron logs for more information. [ 772.989339] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] [ 772.989339] env[62499]: INFO nova.compute.manager [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Terminating instance [ 772.989797] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Acquiring lock "refresh_cache-e7023ea7-9a39-4e5b-90f6-b2499e06ee9d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 773.052705] env[62499]: DEBUG nova.network.neutron [req-1aaa3cc9-3e31-4885-a347-e39222f65a01 req-da7d8a10-3371-4e5f-be63-750ba218f11d service nova] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 773.150139] env[62499]: DEBUG nova.network.neutron [req-1aaa3cc9-3e31-4885-a347-e39222f65a01 req-da7d8a10-3371-4e5f-be63-750ba218f11d service nova] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 773.160589] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 773.349935] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 773.437060] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 773.652951] env[62499]: DEBUG oslo_concurrency.lockutils [req-1aaa3cc9-3e31-4885-a347-e39222f65a01 req-da7d8a10-3371-4e5f-be63-750ba218f11d service nova] Releasing lock "refresh_cache-e7023ea7-9a39-4e5b-90f6-b2499e06ee9d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 773.653363] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Acquired lock "refresh_cache-e7023ea7-9a39-4e5b-90f6-b2499e06ee9d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 773.653542] env[62499]: DEBUG nova.network.neutron [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 773.684267] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8c32b6f-4681-4edf-b6a1-bc7490cd1fd7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 773.691793] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f89b68aa-34b7-449c-a8ce-e8b3e8546a5e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 773.720955] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d48b220-39b3-4b0a-a4ef-b5125c1d0d43 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 773.728078] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-730b47f8-e792-45e8-8dac-107dfd84e9e4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 773.741074] env[62499]: DEBUG nova.compute.provider_tree [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 773.939308] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Releasing lock "refresh_cache-0224e4da-3079-477f-96d8-671985e85c39" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 773.939580] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 773.939807] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 773.939984] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 773.956693] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 774.173180] env[62499]: DEBUG nova.network.neutron [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 774.244916] env[62499]: DEBUG nova.scheduler.client.report [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 774.275620] env[62499]: DEBUG nova.network.neutron [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 774.460537] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 774.546553] env[62499]: DEBUG nova.compute.manager [req-978062cf-8789-459f-afaa-6c135034b368 req-d4470376-8b72-4d6d-b399-83283ab2bc99 service nova] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Received event network-vif-deleted-7b317662-2319-4c6a-8a87-72b6d90c23aa {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 774.750999] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.929s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 774.751639] env[62499]: ERROR nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 87103792-bec8-416e-89d2-1438ea63251a, please check neutron logs for more information. [ 774.751639] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Traceback (most recent call last): [ 774.751639] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 774.751639] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] self.driver.spawn(context, instance, image_meta, [ 774.751639] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 774.751639] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] self._vmops.spawn(context, instance, image_meta, injected_files, [ 774.751639] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 774.751639] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] vm_ref = self.build_virtual_machine(instance, [ 774.751639] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 774.751639] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] vif_infos = vmwarevif.get_vif_info(self._session, [ 774.751639] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 774.752044] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] for vif in network_info: [ 774.752044] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 774.752044] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] return self._sync_wrapper(fn, *args, **kwargs) [ 774.752044] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 774.752044] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] self.wait() [ 774.752044] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 774.752044] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] self[:] = self._gt.wait() [ 774.752044] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 774.752044] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] return self._exit_event.wait() [ 774.752044] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 774.752044] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] current.throw(*self._exc) [ 774.752044] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 774.752044] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] result = function(*args, **kwargs) [ 774.752463] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 774.752463] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] return func(*args, **kwargs) [ 774.752463] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 774.752463] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] raise e [ 774.752463] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 774.752463] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] nwinfo = self.network_api.allocate_for_instance( [ 774.752463] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 774.752463] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] created_port_ids = self._update_ports_for_instance( [ 774.752463] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 774.752463] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] with excutils.save_and_reraise_exception(): [ 774.752463] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 774.752463] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] self.force_reraise() [ 774.752463] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 774.752887] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] raise self.value [ 774.752887] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 774.752887] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] updated_port = self._update_port( [ 774.752887] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 774.752887] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] _ensure_no_port_binding_failure(port) [ 774.752887] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 774.752887] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] raise exception.PortBindingFailed(port_id=port['id']) [ 774.752887] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] nova.exception.PortBindingFailed: Binding failed for port 87103792-bec8-416e-89d2-1438ea63251a, please check neutron logs for more information. [ 774.752887] env[62499]: ERROR nova.compute.manager [instance: 477a1892-0316-42ff-8fc8-269239cb2198] [ 774.752887] env[62499]: DEBUG nova.compute.utils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Binding failed for port 87103792-bec8-416e-89d2-1438ea63251a, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 774.753608] env[62499]: DEBUG oslo_concurrency.lockutils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.280s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 774.755106] env[62499]: INFO nova.compute.claims [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 774.757632] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Build of instance 477a1892-0316-42ff-8fc8-269239cb2198 was re-scheduled: Binding failed for port 87103792-bec8-416e-89d2-1438ea63251a, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 774.758067] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 774.758290] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquiring lock "refresh_cache-477a1892-0316-42ff-8fc8-269239cb2198" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 774.758431] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquired lock "refresh_cache-477a1892-0316-42ff-8fc8-269239cb2198" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 774.758583] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 774.779104] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Releasing lock "refresh_cache-e7023ea7-9a39-4e5b-90f6-b2499e06ee9d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 774.779493] env[62499]: DEBUG nova.compute.manager [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 774.779706] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 774.779990] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-6e9c6fd7-9c32-4296-873f-bab84c545ffa {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 774.789018] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ffa7929d-3d6f-4c1b-bf08-4062cd135a95 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 774.811047] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance e7023ea7-9a39-4e5b-90f6-b2499e06ee9d could not be found. [ 774.811293] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 774.811471] env[62499]: INFO nova.compute.manager [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Took 0.03 seconds to destroy the instance on the hypervisor. [ 774.811712] env[62499]: DEBUG oslo.service.loopingcall [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 774.811918] env[62499]: DEBUG nova.compute.manager [-] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 774.812016] env[62499]: DEBUG nova.network.neutron [-] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 774.831785] env[62499]: DEBUG nova.network.neutron [-] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 774.966234] env[62499]: INFO nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 0224e4da-3079-477f-96d8-671985e85c39] Took 1.03 seconds to deallocate network for instance. [ 775.286661] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 775.338024] env[62499]: DEBUG nova.network.neutron [-] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 775.382017] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 775.839803] env[62499]: INFO nova.compute.manager [-] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Took 1.03 seconds to deallocate network for instance. [ 775.844536] env[62499]: DEBUG nova.compute.claims [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 775.844720] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 775.887815] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Releasing lock "refresh_cache-477a1892-0316-42ff-8fc8-269239cb2198" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 775.888057] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 775.888240] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 775.888405] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 775.955734] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 776.004783] env[62499]: INFO nova.scheduler.client.report [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Deleted allocations for instance 0224e4da-3079-477f-96d8-671985e85c39 [ 776.112476] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-653139d0-75c3-4d26-a61c-315cb35c1eda {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 776.119398] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f921df87-e9a3-4cd9-958f-73c150dd4e52 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 776.149404] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-df8fdb19-b340-4436-ab6f-8cad7047c8c5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 776.156769] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-24fe9677-3311-4142-b102-cd1e0e5a6fef {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 776.170318] env[62499]: DEBUG nova.compute.provider_tree [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 776.460204] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 776.515181] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "0224e4da-3079-477f-96d8-671985e85c39" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 133.235s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 776.674019] env[62499]: DEBUG nova.scheduler.client.report [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 776.965343] env[62499]: INFO nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 477a1892-0316-42ff-8fc8-269239cb2198] Took 1.08 seconds to deallocate network for instance. [ 777.018506] env[62499]: DEBUG nova.compute.manager [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 777.180797] env[62499]: DEBUG oslo_concurrency.lockutils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.426s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 777.180797] env[62499]: DEBUG nova.compute.manager [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 777.183064] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 14.521s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 777.541798] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 777.689418] env[62499]: DEBUG nova.compute.utils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 777.690802] env[62499]: DEBUG nova.compute.manager [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 777.690969] env[62499]: DEBUG nova.network.neutron [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 777.737560] env[62499]: DEBUG nova.policy [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f7f6adeea8054eefaf9d72e4264474c2', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd6e3f664f5d3403289b586dcceb28424', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 777.983945] env[62499]: DEBUG nova.network.neutron [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Successfully created port: 0ae0b0d1-4514-4899-a479-3423a0b56ede {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 777.993896] env[62499]: INFO nova.scheduler.client.report [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Deleted allocations for instance 477a1892-0316-42ff-8fc8-269239cb2198 [ 778.194612] env[62499]: DEBUG nova.compute.manager [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 778.221333] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 399b809e-8926-4348-bfaf-3499de57c1b3 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 778.221491] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 6e078697-9fc5-4605-8d4a-a47d9331ba9c actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 778.221614] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance e7023ea7-9a39-4e5b-90f6-b2499e06ee9d actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 778.221732] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance ea14c163-3afe-4a8d-8370-43e26ba7af37 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 778.509660] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "477a1892-0316-42ff-8fc8-269239cb2198" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 135.192s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 778.729878] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 778.899156] env[62499]: DEBUG nova.compute.manager [req-fe60a411-2001-4351-bfe8-a441706dc3df req-a3fa2b3a-8f61-4a24-9114-2bc62f56d574 service nova] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Received event network-changed-0ae0b0d1-4514-4899-a479-3423a0b56ede {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 778.899364] env[62499]: DEBUG nova.compute.manager [req-fe60a411-2001-4351-bfe8-a441706dc3df req-a3fa2b3a-8f61-4a24-9114-2bc62f56d574 service nova] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Refreshing instance network info cache due to event network-changed-0ae0b0d1-4514-4899-a479-3423a0b56ede. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 778.899575] env[62499]: DEBUG oslo_concurrency.lockutils [req-fe60a411-2001-4351-bfe8-a441706dc3df req-a3fa2b3a-8f61-4a24-9114-2bc62f56d574 service nova] Acquiring lock "refresh_cache-ea14c163-3afe-4a8d-8370-43e26ba7af37" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 778.899782] env[62499]: DEBUG oslo_concurrency.lockutils [req-fe60a411-2001-4351-bfe8-a441706dc3df req-a3fa2b3a-8f61-4a24-9114-2bc62f56d574 service nova] Acquired lock "refresh_cache-ea14c163-3afe-4a8d-8370-43e26ba7af37" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 778.899947] env[62499]: DEBUG nova.network.neutron [req-fe60a411-2001-4351-bfe8-a441706dc3df req-a3fa2b3a-8f61-4a24-9114-2bc62f56d574 service nova] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Refreshing network info cache for port 0ae0b0d1-4514-4899-a479-3423a0b56ede {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 779.016232] env[62499]: DEBUG nova.compute.manager [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 779.086487] env[62499]: ERROR nova.compute.manager [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 0ae0b0d1-4514-4899-a479-3423a0b56ede, please check neutron logs for more information. [ 779.086487] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 779.086487] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 779.086487] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 779.086487] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 779.086487] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 779.086487] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 779.086487] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 779.086487] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 779.086487] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 779.086487] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 779.086487] env[62499]: ERROR nova.compute.manager raise self.value [ 779.086487] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 779.086487] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 779.086487] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 779.086487] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 779.087029] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 779.087029] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 779.087029] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 0ae0b0d1-4514-4899-a479-3423a0b56ede, please check neutron logs for more information. [ 779.087029] env[62499]: ERROR nova.compute.manager [ 779.087029] env[62499]: Traceback (most recent call last): [ 779.087029] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 779.087029] env[62499]: listener.cb(fileno) [ 779.087029] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 779.087029] env[62499]: result = function(*args, **kwargs) [ 779.087029] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 779.087029] env[62499]: return func(*args, **kwargs) [ 779.087029] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 779.087029] env[62499]: raise e [ 779.087029] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 779.087029] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 779.087029] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 779.087029] env[62499]: created_port_ids = self._update_ports_for_instance( [ 779.087029] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 779.087029] env[62499]: with excutils.save_and_reraise_exception(): [ 779.087029] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 779.087029] env[62499]: self.force_reraise() [ 779.087029] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 779.087029] env[62499]: raise self.value [ 779.087029] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 779.087029] env[62499]: updated_port = self._update_port( [ 779.087029] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 779.087029] env[62499]: _ensure_no_port_binding_failure(port) [ 779.087029] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 779.087029] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 779.088367] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 0ae0b0d1-4514-4899-a479-3423a0b56ede, please check neutron logs for more information. [ 779.088367] env[62499]: Removing descriptor: 17 [ 779.204036] env[62499]: DEBUG nova.compute.manager [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 779.230063] env[62499]: DEBUG nova.virt.hardware [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 779.230341] env[62499]: DEBUG nova.virt.hardware [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 779.230498] env[62499]: DEBUG nova.virt.hardware [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 779.230685] env[62499]: DEBUG nova.virt.hardware [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 779.230833] env[62499]: DEBUG nova.virt.hardware [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 779.230976] env[62499]: DEBUG nova.virt.hardware [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 779.231413] env[62499]: DEBUG nova.virt.hardware [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 779.231492] env[62499]: DEBUG nova.virt.hardware [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 779.231680] env[62499]: DEBUG nova.virt.hardware [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 779.231874] env[62499]: DEBUG nova.virt.hardware [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 779.232096] env[62499]: DEBUG nova.virt.hardware [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 779.233121] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d2f3af2f-e2fe-4d56-b5fa-9ee0f989993d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 779.241461] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 7bdd4371-b5d5-4053-93b1-75d1c5b9835b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 779.243699] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cbef5546-1639-4c25-b4be-bfa30276029c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 779.260088] env[62499]: ERROR nova.compute.manager [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 0ae0b0d1-4514-4899-a479-3423a0b56ede, please check neutron logs for more information. [ 779.260088] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Traceback (most recent call last): [ 779.260088] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 779.260088] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] yield resources [ 779.260088] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 779.260088] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] self.driver.spawn(context, instance, image_meta, [ 779.260088] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 779.260088] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] self._vmops.spawn(context, instance, image_meta, injected_files, [ 779.260088] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 779.260088] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] vm_ref = self.build_virtual_machine(instance, [ 779.260088] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 779.260582] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] vif_infos = vmwarevif.get_vif_info(self._session, [ 779.260582] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 779.260582] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] for vif in network_info: [ 779.260582] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 779.260582] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] return self._sync_wrapper(fn, *args, **kwargs) [ 779.260582] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 779.260582] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] self.wait() [ 779.260582] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 779.260582] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] self[:] = self._gt.wait() [ 779.260582] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 779.260582] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] return self._exit_event.wait() [ 779.260582] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 779.260582] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] current.throw(*self._exc) [ 779.260993] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 779.260993] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] result = function(*args, **kwargs) [ 779.260993] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 779.260993] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] return func(*args, **kwargs) [ 779.260993] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 779.260993] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] raise e [ 779.260993] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 779.260993] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] nwinfo = self.network_api.allocate_for_instance( [ 779.260993] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 779.260993] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] created_port_ids = self._update_ports_for_instance( [ 779.260993] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 779.260993] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] with excutils.save_and_reraise_exception(): [ 779.260993] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 779.261419] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] self.force_reraise() [ 779.261419] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 779.261419] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] raise self.value [ 779.261419] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 779.261419] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] updated_port = self._update_port( [ 779.261419] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 779.261419] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] _ensure_no_port_binding_failure(port) [ 779.261419] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 779.261419] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] raise exception.PortBindingFailed(port_id=port['id']) [ 779.261419] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] nova.exception.PortBindingFailed: Binding failed for port 0ae0b0d1-4514-4899-a479-3423a0b56ede, please check neutron logs for more information. [ 779.261419] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] [ 779.261419] env[62499]: INFO nova.compute.manager [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Terminating instance [ 779.262561] env[62499]: DEBUG oslo_concurrency.lockutils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Acquiring lock "refresh_cache-ea14c163-3afe-4a8d-8370-43e26ba7af37" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 779.418022] env[62499]: DEBUG nova.network.neutron [req-fe60a411-2001-4351-bfe8-a441706dc3df req-a3fa2b3a-8f61-4a24-9114-2bc62f56d574 service nova] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 779.517330] env[62499]: DEBUG nova.network.neutron [req-fe60a411-2001-4351-bfe8-a441706dc3df req-a3fa2b3a-8f61-4a24-9114-2bc62f56d574 service nova] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 779.541741] env[62499]: DEBUG oslo_concurrency.lockutils [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 779.630522] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "a92509e3-d5b1-41d3-b9af-fa0af8a956c9" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 779.630801] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "a92509e3-d5b1-41d3-b9af-fa0af8a956c9" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 779.749547] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 63b76c39-11ca-4f1e-b336-2caceece1f72 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 780.022183] env[62499]: DEBUG oslo_concurrency.lockutils [req-fe60a411-2001-4351-bfe8-a441706dc3df req-a3fa2b3a-8f61-4a24-9114-2bc62f56d574 service nova] Releasing lock "refresh_cache-ea14c163-3afe-4a8d-8370-43e26ba7af37" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 780.022584] env[62499]: DEBUG oslo_concurrency.lockutils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Acquired lock "refresh_cache-ea14c163-3afe-4a8d-8370-43e26ba7af37" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 780.022884] env[62499]: DEBUG nova.network.neutron [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 780.252455] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 008b230e-72b8-43ae-826a-b38111c56e76 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 780.543492] env[62499]: DEBUG nova.network.neutron [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 780.637209] env[62499]: DEBUG nova.network.neutron [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 780.758737] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 39bc8b3a-e9d9-48e3-a193-5fbdc2454346 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 780.938618] env[62499]: DEBUG nova.compute.manager [req-5490d97b-53e2-4362-9f18-b64a0822f03f req-6a4371fd-7529-4332-877c-eafc817f95cc service nova] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Received event network-vif-deleted-0ae0b0d1-4514-4899-a479-3423a0b56ede {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 781.139991] env[62499]: DEBUG oslo_concurrency.lockutils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Releasing lock "refresh_cache-ea14c163-3afe-4a8d-8370-43e26ba7af37" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 781.140465] env[62499]: DEBUG nova.compute.manager [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 781.140661] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 781.140963] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-111f37a0-7a36-4a21-b319-09ecc2e2058c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 781.149953] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7d3996a-23c7-4a30-a6a8-debba14e4f3b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 781.169932] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ea14c163-3afe-4a8d-8370-43e26ba7af37 could not be found. [ 781.170158] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 781.170341] env[62499]: INFO nova.compute.manager [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Took 0.03 seconds to destroy the instance on the hypervisor. [ 781.170582] env[62499]: DEBUG oslo.service.loopingcall [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 781.170786] env[62499]: DEBUG nova.compute.manager [-] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 781.170879] env[62499]: DEBUG nova.network.neutron [-] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 781.183955] env[62499]: DEBUG nova.network.neutron [-] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 781.262589] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 15f8e04f-8c14-4315-8686-da4db517f7b6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 781.686606] env[62499]: DEBUG nova.network.neutron [-] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 781.764846] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 57d21761-431b-4a41-86eb-038f8c35d8ef has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 782.190290] env[62499]: INFO nova.compute.manager [-] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Took 1.02 seconds to deallocate network for instance. [ 782.193502] env[62499]: DEBUG nova.compute.claims [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 782.193502] env[62499]: DEBUG oslo_concurrency.lockutils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 782.268950] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 261cbefa-fb2d-48da-a4fe-80b744a931f6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 782.772425] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 49f9108c-7256-4a2c-9ffd-a6d041a180e4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 783.275897] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance d650a1a5-3706-4682-a813-f85ea23098e2 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 783.779884] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 60a4b8c3-9dc4-4867-adca-4503e61ce237 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 784.283073] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance ad749135-08c1-48ca-af56-300d6e796012 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 784.785583] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 2a1c52ab-8ee6-4b63-ad04-2450a60262d0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 785.288731] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 6da95995-0527-4b05-94dc-a4fcf0ae2f92 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 785.792329] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 263931cd-b2dc-41bb-8a2b-abf61aadafc9 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 786.295754] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 20e4ff1b-0b84-477d-a0d0-a85a439449ad has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 786.799179] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 4eda80db-5db7-4fd6-9932-108c22f4a616 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 787.303050] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance c1961f60-06d8-44d8-999f-d498c3667adc has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 787.805525] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance ba8fc3e0-dae8-4910-858b-dab1de72089e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 788.308911] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 0eaadb76-e2c7-4e06-b865-065ab0f014ba has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 256, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 788.811904] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 764ef9ab-22ae-45cd-8b5d-76a63346d93c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 788.812203] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Total usable vcpus: 48, total allocated vcpus: 4 {{(pid=62499) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 788.812366] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1280MB phys_disk=200GB used_disk=4GB total_vcpus=48 used_vcpus=4 pci_stats=[] {{(pid=62499) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 789.102389] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5bdddb55-c9a7-4a41-ac93-dce556a56006 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 789.110915] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ecbc06f-9db0-44fc-a957-958b8b0d94ea {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 789.140492] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5069c4f-a222-415c-a021-f4a9a6ef2c86 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 789.147353] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fcaea303-3198-4fc2-9e03-ff17ba81098f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 789.160046] env[62499]: DEBUG nova.compute.provider_tree [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 789.663151] env[62499]: DEBUG nova.scheduler.client.report [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 790.168122] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62499) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 790.168410] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 12.985s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 790.168686] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 26.904s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 790.171738] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 790.171906] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Cleaning up deleted instances {{(pid=62499) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11228}} [ 790.677809] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] There are 2 instances to clean {{(pid=62499) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11237}} [ 790.678142] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 27103ce2-0962-4fce-9331-d74179b0510c] Instance has had 0 of 5 cleanup attempts {{(pid=62499) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11241}} [ 790.963455] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7025080c-2553-4c1d-a4f3-db717639d77b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 790.971147] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85417da5-4b8d-41d9-9284-cf5b57e78032 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 791.000502] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c07469f-69c6-4b68-b3c2-4cfbfd58e635 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 791.010761] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-10770c2d-1cc4-422d-9e77-89059cc6f2e5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 791.020947] env[62499]: DEBUG nova.compute.provider_tree [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 791.181912] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: ddd212aa-9f4a-4b9d-aa60-cd4b64580f84] Instance has had 0 of 5 cleanup attempts {{(pid=62499) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11241}} [ 791.524681] env[62499]: DEBUG nova.scheduler.client.report [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 791.685347] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 791.685347] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Cleaning up deleted instances with incomplete migration {{(pid=62499) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11266}} [ 792.030117] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.860s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 792.030117] env[62499]: ERROR nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 7aef66d5-4e71-41da-a7f8-5ea10eb75abd, please check neutron logs for more information. [ 792.030117] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Traceback (most recent call last): [ 792.030117] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 792.030117] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] self.driver.spawn(context, instance, image_meta, [ 792.030117] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 792.030117] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 792.030117] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 792.030117] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] vm_ref = self.build_virtual_machine(instance, [ 792.030455] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 792.030455] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] vif_infos = vmwarevif.get_vif_info(self._session, [ 792.030455] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 792.030455] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] for vif in network_info: [ 792.030455] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 792.030455] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] return self._sync_wrapper(fn, *args, **kwargs) [ 792.030455] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 792.030455] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] self.wait() [ 792.030455] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 792.030455] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] self[:] = self._gt.wait() [ 792.030455] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 792.030455] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] return self._exit_event.wait() [ 792.030455] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 792.030789] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] current.throw(*self._exc) [ 792.030789] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 792.030789] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] result = function(*args, **kwargs) [ 792.030789] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 792.030789] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] return func(*args, **kwargs) [ 792.030789] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 792.030789] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] raise e [ 792.030789] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 792.030789] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] nwinfo = self.network_api.allocate_for_instance( [ 792.030789] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 792.030789] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] created_port_ids = self._update_ports_for_instance( [ 792.030789] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 792.030789] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] with excutils.save_and_reraise_exception(): [ 792.031115] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 792.031115] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] self.force_reraise() [ 792.031115] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 792.031115] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] raise self.value [ 792.031115] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 792.031115] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] updated_port = self._update_port( [ 792.031115] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 792.031115] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] _ensure_no_port_binding_failure(port) [ 792.031115] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 792.031115] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] raise exception.PortBindingFailed(port_id=port['id']) [ 792.031115] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] nova.exception.PortBindingFailed: Binding failed for port 7aef66d5-4e71-41da-a7f8-5ea10eb75abd, please check neutron logs for more information. [ 792.031115] env[62499]: ERROR nova.compute.manager [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] [ 792.031547] env[62499]: DEBUG nova.compute.utils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Binding failed for port 7aef66d5-4e71-41da-a7f8-5ea10eb75abd, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 792.032146] env[62499]: DEBUG oslo_concurrency.lockutils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 27.149s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 792.033430] env[62499]: INFO nova.compute.claims [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 792.035925] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Build of instance 399b809e-8926-4348-bfaf-3499de57c1b3 was re-scheduled: Binding failed for port 7aef66d5-4e71-41da-a7f8-5ea10eb75abd, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 792.036904] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 792.036904] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquiring lock "refresh_cache-399b809e-8926-4348-bfaf-3499de57c1b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 792.036904] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Acquired lock "refresh_cache-399b809e-8926-4348-bfaf-3499de57c1b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 792.036904] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 792.187656] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 792.558756] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 792.636390] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 793.138570] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Releasing lock "refresh_cache-399b809e-8926-4348-bfaf-3499de57c1b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 793.138825] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 793.139017] env[62499]: DEBUG nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 793.139196] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 793.154341] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 793.329359] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75336aa5-875c-46be-9cb5-86da772a79a3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 793.336788] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-22968fb6-090b-4eb4-ae21-ad61fce54537 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 793.364993] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1aaad05d-2dda-4103-9101-07039d3db335 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 793.371641] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8eed5f92-25e3-494a-aafc-13c6fd4931a8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 793.384326] env[62499]: DEBUG nova.compute.provider_tree [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 793.657727] env[62499]: DEBUG nova.network.neutron [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 793.887662] env[62499]: DEBUG nova.scheduler.client.report [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 794.159957] env[62499]: INFO nova.compute.manager [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] [instance: 399b809e-8926-4348-bfaf-3499de57c1b3] Took 1.02 seconds to deallocate network for instance. [ 794.393085] env[62499]: DEBUG oslo_concurrency.lockutils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.361s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 794.393465] env[62499]: DEBUG nova.compute.manager [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 794.395934] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 26.858s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 794.900705] env[62499]: DEBUG nova.compute.utils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 794.904801] env[62499]: DEBUG nova.compute.manager [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 794.904972] env[62499]: DEBUG nova.network.neutron [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 794.964920] env[62499]: DEBUG nova.policy [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '30a7d1e49988462ab49214182fc00e5e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '26789909db1d4930acc0b87358ba159d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 795.187069] env[62499]: INFO nova.scheduler.client.report [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Deleted allocations for instance 399b809e-8926-4348-bfaf-3499de57c1b3 [ 795.304227] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2bf0177d-740e-4a3d-a2b9-c220c33cfd22 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 795.312043] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67592469-ac62-4f1e-8511-9bed3d21f925 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 795.343708] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-856aadca-1275-4ea7-909d-eb7ecc355e01 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 795.353299] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-351d121d-90f2-419e-9980-a00163ec8123 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 795.367437] env[62499]: DEBUG nova.compute.provider_tree [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 795.410955] env[62499]: DEBUG nova.compute.manager [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 795.507487] env[62499]: DEBUG nova.network.neutron [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Successfully created port: 2e2b4e94-b7e3-44ca-85a2-06de83bce3e0 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 795.701022] env[62499]: DEBUG oslo_concurrency.lockutils [None req-460c06c1-91d0-4b94-a895-15168827dcea tempest-ListServersNegativeTestJSON-1227509665 tempest-ListServersNegativeTestJSON-1227509665-project-member] Lock "399b809e-8926-4348-bfaf-3499de57c1b3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 152.350s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 795.871677] env[62499]: DEBUG nova.scheduler.client.report [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 796.202609] env[62499]: DEBUG nova.compute.manager [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 796.206872] env[62499]: DEBUG nova.compute.manager [req-59b37b19-3b7e-4692-902d-331a7821ee71 req-a81a638d-f727-459d-af01-d54f46f06b92 service nova] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Received event network-changed-2e2b4e94-b7e3-44ca-85a2-06de83bce3e0 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 796.207077] env[62499]: DEBUG nova.compute.manager [req-59b37b19-3b7e-4692-902d-331a7821ee71 req-a81a638d-f727-459d-af01-d54f46f06b92 service nova] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Refreshing instance network info cache due to event network-changed-2e2b4e94-b7e3-44ca-85a2-06de83bce3e0. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 796.207295] env[62499]: DEBUG oslo_concurrency.lockutils [req-59b37b19-3b7e-4692-902d-331a7821ee71 req-a81a638d-f727-459d-af01-d54f46f06b92 service nova] Acquiring lock "refresh_cache-6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 796.207427] env[62499]: DEBUG oslo_concurrency.lockutils [req-59b37b19-3b7e-4692-902d-331a7821ee71 req-a81a638d-f727-459d-af01-d54f46f06b92 service nova] Acquired lock "refresh_cache-6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 796.207594] env[62499]: DEBUG nova.network.neutron [req-59b37b19-3b7e-4692-902d-331a7821ee71 req-a81a638d-f727-459d-af01-d54f46f06b92 service nova] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Refreshing network info cache for port 2e2b4e94-b7e3-44ca-85a2-06de83bce3e0 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 796.377683] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.981s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 796.378488] env[62499]: ERROR nova.compute.manager [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 8d274454-fb43-45a5-9745-eda3d5e00015, please check neutron logs for more information. [ 796.378488] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Traceback (most recent call last): [ 796.378488] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 796.378488] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] self.driver.spawn(context, instance, image_meta, [ 796.378488] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 796.378488] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 796.378488] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 796.378488] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] vm_ref = self.build_virtual_machine(instance, [ 796.378488] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 796.378488] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] vif_infos = vmwarevif.get_vif_info(self._session, [ 796.378488] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 796.378792] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] for vif in network_info: [ 796.378792] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 796.378792] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] return self._sync_wrapper(fn, *args, **kwargs) [ 796.378792] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 796.378792] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] self.wait() [ 796.378792] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 796.378792] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] self[:] = self._gt.wait() [ 796.378792] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 796.378792] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] return self._exit_event.wait() [ 796.378792] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 796.378792] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] current.throw(*self._exc) [ 796.378792] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 796.378792] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] result = function(*args, **kwargs) [ 796.379065] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 796.379065] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] return func(*args, **kwargs) [ 796.379065] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 796.379065] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] raise e [ 796.379065] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 796.379065] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] nwinfo = self.network_api.allocate_for_instance( [ 796.379065] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 796.379065] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] created_port_ids = self._update_ports_for_instance( [ 796.379065] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 796.379065] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] with excutils.save_and_reraise_exception(): [ 796.379065] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 796.379065] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] self.force_reraise() [ 796.379065] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 796.379394] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] raise self.value [ 796.379394] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 796.379394] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] updated_port = self._update_port( [ 796.379394] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 796.379394] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] _ensure_no_port_binding_failure(port) [ 796.379394] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 796.379394] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] raise exception.PortBindingFailed(port_id=port['id']) [ 796.379394] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] nova.exception.PortBindingFailed: Binding failed for port 8d274454-fb43-45a5-9745-eda3d5e00015, please check neutron logs for more information. [ 796.379394] env[62499]: ERROR nova.compute.manager [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] [ 796.379394] env[62499]: DEBUG nova.compute.utils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Binding failed for port 8d274454-fb43-45a5-9745-eda3d5e00015, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 796.380644] env[62499]: DEBUG nova.compute.manager [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Build of instance 6e078697-9fc5-4605-8d4a-a47d9331ba9c was re-scheduled: Binding failed for port 8d274454-fb43-45a5-9745-eda3d5e00015, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 796.381168] env[62499]: DEBUG nova.compute.manager [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 796.381298] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquiring lock "refresh_cache-6e078697-9fc5-4605-8d4a-a47d9331ba9c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 796.381465] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquired lock "refresh_cache-6e078697-9fc5-4605-8d4a-a47d9331ba9c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 796.381626] env[62499]: DEBUG nova.network.neutron [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 796.383019] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 27.087s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 796.385042] env[62499]: INFO nova.compute.claims [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 796.420472] env[62499]: DEBUG nova.compute.manager [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 796.453912] env[62499]: DEBUG nova.virt.hardware [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 796.453912] env[62499]: DEBUG nova.virt.hardware [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 796.453912] env[62499]: DEBUG nova.virt.hardware [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 796.453912] env[62499]: DEBUG nova.virt.hardware [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 796.454312] env[62499]: DEBUG nova.virt.hardware [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 796.454550] env[62499]: DEBUG nova.virt.hardware [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 796.454840] env[62499]: DEBUG nova.virt.hardware [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 796.455499] env[62499]: DEBUG nova.virt.hardware [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 796.455499] env[62499]: DEBUG nova.virt.hardware [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 796.455499] env[62499]: DEBUG nova.virt.hardware [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 796.455736] env[62499]: DEBUG nova.virt.hardware [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 796.456654] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f978d90e-5bb6-4fcb-b766-19c3f16c7440 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.465842] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae86d79b-13cc-4dec-afbb-29e33fa2624c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.481261] env[62499]: ERROR nova.compute.manager [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 2e2b4e94-b7e3-44ca-85a2-06de83bce3e0, please check neutron logs for more information. [ 796.481261] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 796.481261] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 796.481261] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 796.481261] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 796.481261] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 796.481261] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 796.481261] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 796.481261] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 796.481261] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 796.481261] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 796.481261] env[62499]: ERROR nova.compute.manager raise self.value [ 796.481261] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 796.481261] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 796.481261] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 796.481261] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 796.481709] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 796.481709] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 796.481709] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 2e2b4e94-b7e3-44ca-85a2-06de83bce3e0, please check neutron logs for more information. [ 796.481709] env[62499]: ERROR nova.compute.manager [ 796.481709] env[62499]: Traceback (most recent call last): [ 796.481709] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 796.481709] env[62499]: listener.cb(fileno) [ 796.481709] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 796.481709] env[62499]: result = function(*args, **kwargs) [ 796.481709] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 796.481709] env[62499]: return func(*args, **kwargs) [ 796.481709] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 796.481709] env[62499]: raise e [ 796.481709] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 796.481709] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 796.481709] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 796.481709] env[62499]: created_port_ids = self._update_ports_for_instance( [ 796.481709] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 796.481709] env[62499]: with excutils.save_and_reraise_exception(): [ 796.481709] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 796.481709] env[62499]: self.force_reraise() [ 796.481709] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 796.481709] env[62499]: raise self.value [ 796.481709] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 796.481709] env[62499]: updated_port = self._update_port( [ 796.481709] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 796.481709] env[62499]: _ensure_no_port_binding_failure(port) [ 796.481709] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 796.481709] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 796.482447] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 2e2b4e94-b7e3-44ca-85a2-06de83bce3e0, please check neutron logs for more information. [ 796.482447] env[62499]: Removing descriptor: 17 [ 796.482447] env[62499]: ERROR nova.compute.manager [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 2e2b4e94-b7e3-44ca-85a2-06de83bce3e0, please check neutron logs for more information. [ 796.482447] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Traceback (most recent call last): [ 796.482447] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 796.482447] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] yield resources [ 796.482447] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 796.482447] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] self.driver.spawn(context, instance, image_meta, [ 796.482447] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 796.482447] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 796.482447] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 796.482447] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] vm_ref = self.build_virtual_machine(instance, [ 796.482842] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 796.482842] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] vif_infos = vmwarevif.get_vif_info(self._session, [ 796.482842] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 796.482842] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] for vif in network_info: [ 796.482842] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 796.482842] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] return self._sync_wrapper(fn, *args, **kwargs) [ 796.482842] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 796.482842] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] self.wait() [ 796.482842] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 796.482842] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] self[:] = self._gt.wait() [ 796.482842] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 796.482842] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] return self._exit_event.wait() [ 796.482842] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 796.483173] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] result = hub.switch() [ 796.483173] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 796.483173] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] return self.greenlet.switch() [ 796.483173] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 796.483173] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] result = function(*args, **kwargs) [ 796.483173] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 796.483173] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] return func(*args, **kwargs) [ 796.483173] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 796.483173] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] raise e [ 796.483173] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 796.483173] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] nwinfo = self.network_api.allocate_for_instance( [ 796.483173] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 796.483173] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] created_port_ids = self._update_ports_for_instance( [ 796.483507] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 796.483507] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] with excutils.save_and_reraise_exception(): [ 796.483507] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 796.483507] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] self.force_reraise() [ 796.483507] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 796.483507] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] raise self.value [ 796.483507] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 796.483507] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] updated_port = self._update_port( [ 796.483507] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 796.483507] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] _ensure_no_port_binding_failure(port) [ 796.483507] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 796.483507] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] raise exception.PortBindingFailed(port_id=port['id']) [ 796.483822] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] nova.exception.PortBindingFailed: Binding failed for port 2e2b4e94-b7e3-44ca-85a2-06de83bce3e0, please check neutron logs for more information. [ 796.483822] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] [ 796.483822] env[62499]: INFO nova.compute.manager [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Terminating instance [ 796.485077] env[62499]: DEBUG oslo_concurrency.lockutils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Acquiring lock "refresh_cache-6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 796.730348] env[62499]: DEBUG nova.network.neutron [req-59b37b19-3b7e-4692-902d-331a7821ee71 req-a81a638d-f727-459d-af01-d54f46f06b92 service nova] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 796.732760] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 796.790639] env[62499]: DEBUG nova.network.neutron [req-59b37b19-3b7e-4692-902d-331a7821ee71 req-a81a638d-f727-459d-af01-d54f46f06b92 service nova] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 796.907758] env[62499]: DEBUG nova.network.neutron [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 796.982326] env[62499]: DEBUG nova.network.neutron [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 797.293494] env[62499]: DEBUG oslo_concurrency.lockutils [req-59b37b19-3b7e-4692-902d-331a7821ee71 req-a81a638d-f727-459d-af01-d54f46f06b92 service nova] Releasing lock "refresh_cache-6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 797.293911] env[62499]: DEBUG oslo_concurrency.lockutils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Acquired lock "refresh_cache-6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 797.294105] env[62499]: DEBUG nova.network.neutron [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 797.485189] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Releasing lock "refresh_cache-6e078697-9fc5-4605-8d4a-a47d9331ba9c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 797.485423] env[62499]: DEBUG nova.compute.manager [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 797.485600] env[62499]: DEBUG nova.compute.manager [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 797.485825] env[62499]: DEBUG nova.network.neutron [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 797.505828] env[62499]: DEBUG nova.network.neutron [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 797.769015] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96a7d54a-c5cf-4b17-97b4-8c476e163ab4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.780029] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a60a47cd-85bc-4bc8-b141-804baf02c52e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.821707] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da37ae98-e8c3-4667-bf34-9ffcdd953181 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.830474] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7cc2cd76-2241-4864-8a1a-d046fe26e523 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.846119] env[62499]: DEBUG nova.compute.provider_tree [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 797.848727] env[62499]: DEBUG nova.network.neutron [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 797.928947] env[62499]: DEBUG nova.network.neutron [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 798.010124] env[62499]: DEBUG nova.network.neutron [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 798.240474] env[62499]: DEBUG nova.compute.manager [req-1676b810-dd2e-4c3b-8267-7dee01c4a612 req-75e5a9cc-3ccc-4eb0-9b9b-0913d72c26bb service nova] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Received event network-vif-deleted-2e2b4e94-b7e3-44ca-85a2-06de83bce3e0 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 798.352913] env[62499]: DEBUG nova.scheduler.client.report [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 798.431212] env[62499]: DEBUG oslo_concurrency.lockutils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Releasing lock "refresh_cache-6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 798.431656] env[62499]: DEBUG nova.compute.manager [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 798.432087] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 798.432208] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8550b952-2399-44ec-b61b-f8c77d22184d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 798.440865] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-57aaa16d-2579-44d7-b609-6ab91cfea3bc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 798.464028] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5 could not be found. [ 798.464264] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 798.464446] env[62499]: INFO nova.compute.manager [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Took 0.03 seconds to destroy the instance on the hypervisor. [ 798.464695] env[62499]: DEBUG oslo.service.loopingcall [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 798.464899] env[62499]: DEBUG nova.compute.manager [-] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 798.465020] env[62499]: DEBUG nova.network.neutron [-] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 798.487666] env[62499]: DEBUG nova.network.neutron [-] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 798.512296] env[62499]: INFO nova.compute.manager [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 6e078697-9fc5-4605-8d4a-a47d9331ba9c] Took 1.03 seconds to deallocate network for instance. [ 798.861158] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.478s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 798.861533] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 798.864653] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 27.637s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 798.868405] env[62499]: INFO nova.compute.claims [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 798.993164] env[62499]: DEBUG nova.network.neutron [-] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 799.372461] env[62499]: DEBUG nova.compute.utils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 799.378866] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 799.379096] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 799.446141] env[62499]: DEBUG nova.policy [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd896dc782fdf4c13b943c0336d255452', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7fa50ab2c6ba4659afee1f146822a981', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 799.495692] env[62499]: INFO nova.compute.manager [-] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Took 1.03 seconds to deallocate network for instance. [ 799.498606] env[62499]: DEBUG nova.compute.claims [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 799.498789] env[62499]: DEBUG oslo_concurrency.lockutils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 799.546986] env[62499]: INFO nova.scheduler.client.report [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Deleted allocations for instance 6e078697-9fc5-4605-8d4a-a47d9331ba9c [ 799.879656] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 799.951853] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Successfully created port: 1bd118fa-8a80-4b4f-bf10-f9dc71dae0bb {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 800.057921] env[62499]: DEBUG oslo_concurrency.lockutils [None req-a1f3e394-71e3-47cd-ac1c-b6f0d72e7ecd tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "6e078697-9fc5-4605-8d4a-a47d9331ba9c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 156.149s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.338112] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08319614-6c1e-496b-9cda-690456179cdb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 800.348289] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da8afac2-0ac0-4ccc-92cf-db1be9e67eb7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 800.378538] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-43fb5dc0-c517-4c64-90ad-328d5c977e61 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 800.386673] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-793478a3-7b3f-4abe-9832-bc07fdfd1042 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 800.414172] env[62499]: DEBUG nova.compute.provider_tree [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 800.568685] env[62499]: DEBUG nova.compute.manager [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 800.896659] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 800.917799] env[62499]: DEBUG nova.scheduler.client.report [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 800.932105] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 800.932385] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 800.932547] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 800.932725] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 800.932869] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 800.933096] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 800.933396] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 800.933610] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 800.934118] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 800.934373] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 800.934597] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 800.935660] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7cbed63-e79a-4938-ac8a-a7ad206f5898 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 800.950084] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50df864a-7393-409d-a94d-9cad7ccb466e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.096300] env[62499]: DEBUG oslo_concurrency.lockutils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 801.333313] env[62499]: DEBUG nova.compute.manager [req-a6142a5b-3c7a-4102-8e51-2f119363cf75 req-8c386ccf-818a-4256-b53b-e3c307852036 service nova] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Received event network-changed-1bd118fa-8a80-4b4f-bf10-f9dc71dae0bb {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 801.333514] env[62499]: DEBUG nova.compute.manager [req-a6142a5b-3c7a-4102-8e51-2f119363cf75 req-8c386ccf-818a-4256-b53b-e3c307852036 service nova] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Refreshing instance network info cache due to event network-changed-1bd118fa-8a80-4b4f-bf10-f9dc71dae0bb. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 801.333733] env[62499]: DEBUG oslo_concurrency.lockutils [req-a6142a5b-3c7a-4102-8e51-2f119363cf75 req-8c386ccf-818a-4256-b53b-e3c307852036 service nova] Acquiring lock "refresh_cache-7bdd4371-b5d5-4053-93b1-75d1c5b9835b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 801.333865] env[62499]: DEBUG oslo_concurrency.lockutils [req-a6142a5b-3c7a-4102-8e51-2f119363cf75 req-8c386ccf-818a-4256-b53b-e3c307852036 service nova] Acquired lock "refresh_cache-7bdd4371-b5d5-4053-93b1-75d1c5b9835b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 801.334324] env[62499]: DEBUG nova.network.neutron [req-a6142a5b-3c7a-4102-8e51-2f119363cf75 req-8c386ccf-818a-4256-b53b-e3c307852036 service nova] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Refreshing network info cache for port 1bd118fa-8a80-4b4f-bf10-f9dc71dae0bb {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 801.426198] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.562s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 801.426768] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 801.429902] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 28.269s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 801.431727] env[62499]: INFO nova.compute.claims [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 801.603092] env[62499]: ERROR nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1bd118fa-8a80-4b4f-bf10-f9dc71dae0bb, please check neutron logs for more information. [ 801.603092] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 801.603092] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 801.603092] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 801.603092] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 801.603092] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 801.603092] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 801.603092] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 801.603092] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 801.603092] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 801.603092] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 801.603092] env[62499]: ERROR nova.compute.manager raise self.value [ 801.603092] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 801.603092] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 801.603092] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 801.603092] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 801.603507] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 801.603507] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 801.603507] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1bd118fa-8a80-4b4f-bf10-f9dc71dae0bb, please check neutron logs for more information. [ 801.603507] env[62499]: ERROR nova.compute.manager [ 801.603507] env[62499]: Traceback (most recent call last): [ 801.603507] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 801.603507] env[62499]: listener.cb(fileno) [ 801.603507] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 801.603507] env[62499]: result = function(*args, **kwargs) [ 801.603507] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 801.603507] env[62499]: return func(*args, **kwargs) [ 801.603507] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 801.603507] env[62499]: raise e [ 801.603507] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 801.603507] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 801.603507] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 801.603507] env[62499]: created_port_ids = self._update_ports_for_instance( [ 801.603507] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 801.603507] env[62499]: with excutils.save_and_reraise_exception(): [ 801.603507] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 801.603507] env[62499]: self.force_reraise() [ 801.603507] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 801.603507] env[62499]: raise self.value [ 801.603507] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 801.603507] env[62499]: updated_port = self._update_port( [ 801.603507] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 801.603507] env[62499]: _ensure_no_port_binding_failure(port) [ 801.603507] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 801.603507] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 801.604166] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 1bd118fa-8a80-4b4f-bf10-f9dc71dae0bb, please check neutron logs for more information. [ 801.604166] env[62499]: Removing descriptor: 17 [ 801.604166] env[62499]: ERROR nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1bd118fa-8a80-4b4f-bf10-f9dc71dae0bb, please check neutron logs for more information. [ 801.604166] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Traceback (most recent call last): [ 801.604166] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 801.604166] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] yield resources [ 801.604166] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 801.604166] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] self.driver.spawn(context, instance, image_meta, [ 801.604166] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 801.604166] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 801.604166] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 801.604166] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] vm_ref = self.build_virtual_machine(instance, [ 801.604551] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 801.604551] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] vif_infos = vmwarevif.get_vif_info(self._session, [ 801.604551] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 801.604551] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] for vif in network_info: [ 801.604551] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 801.604551] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] return self._sync_wrapper(fn, *args, **kwargs) [ 801.604551] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 801.604551] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] self.wait() [ 801.604551] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 801.604551] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] self[:] = self._gt.wait() [ 801.604551] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 801.604551] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] return self._exit_event.wait() [ 801.604551] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 801.604883] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] result = hub.switch() [ 801.604883] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 801.604883] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] return self.greenlet.switch() [ 801.604883] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 801.604883] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] result = function(*args, **kwargs) [ 801.604883] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 801.604883] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] return func(*args, **kwargs) [ 801.604883] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 801.604883] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] raise e [ 801.604883] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 801.604883] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] nwinfo = self.network_api.allocate_for_instance( [ 801.604883] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 801.604883] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] created_port_ids = self._update_ports_for_instance( [ 801.605645] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 801.605645] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] with excutils.save_and_reraise_exception(): [ 801.605645] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 801.605645] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] self.force_reraise() [ 801.605645] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 801.605645] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] raise self.value [ 801.605645] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 801.605645] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] updated_port = self._update_port( [ 801.605645] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 801.605645] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] _ensure_no_port_binding_failure(port) [ 801.605645] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 801.605645] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] raise exception.PortBindingFailed(port_id=port['id']) [ 801.605951] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] nova.exception.PortBindingFailed: Binding failed for port 1bd118fa-8a80-4b4f-bf10-f9dc71dae0bb, please check neutron logs for more information. [ 801.605951] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] [ 801.605951] env[62499]: INFO nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Terminating instance [ 801.609870] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "refresh_cache-7bdd4371-b5d5-4053-93b1-75d1c5b9835b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 801.858113] env[62499]: DEBUG nova.network.neutron [req-a6142a5b-3c7a-4102-8e51-2f119363cf75 req-8c386ccf-818a-4256-b53b-e3c307852036 service nova] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 801.939674] env[62499]: DEBUG nova.compute.utils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 801.941240] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 801.941400] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 801.963805] env[62499]: DEBUG nova.network.neutron [req-a6142a5b-3c7a-4102-8e51-2f119363cf75 req-8c386ccf-818a-4256-b53b-e3c307852036 service nova] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 802.054723] env[62499]: DEBUG nova.policy [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd896dc782fdf4c13b943c0336d255452', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7fa50ab2c6ba4659afee1f146822a981', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 802.449048] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 802.469718] env[62499]: DEBUG oslo_concurrency.lockutils [req-a6142a5b-3c7a-4102-8e51-2f119363cf75 req-8c386ccf-818a-4256-b53b-e3c307852036 service nova] Releasing lock "refresh_cache-7bdd4371-b5d5-4053-93b1-75d1c5b9835b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 802.470190] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquired lock "refresh_cache-7bdd4371-b5d5-4053-93b1-75d1c5b9835b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 802.470406] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 802.485732] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Successfully created port: 818ca52d-49e7-4e46-a1ac-50d975693883 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 802.810632] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-514c2d2a-b2c4-4a85-a069-114cc15f29f8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 802.819951] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3faac48-4027-4642-b655-a53a75d87cab {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 802.855157] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-48b7a630-3bd1-46be-b2a5-4f2235d37ba7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 802.862625] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3852b8ed-ab55-431d-be41-3aba9834f0eb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 802.876293] env[62499]: DEBUG nova.compute.provider_tree [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 802.996904] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 803.121725] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 803.379130] env[62499]: DEBUG nova.scheduler.client.report [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 803.459141] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 803.493133] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 803.493415] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 803.493538] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 803.493719] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 803.493866] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 803.494081] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 803.494363] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 803.494561] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 803.494897] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 803.495088] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 803.495336] env[62499]: DEBUG nova.virt.hardware [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 803.496177] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5039a1c6-10f7-4155-8a5b-5cb36600dc02 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.504446] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d4da1783-32d0-4d9f-9ab3-8acfd78b1379 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.512767] env[62499]: DEBUG nova.compute.manager [req-eb11bcb9-1f36-4ff4-a0f4-9f2a62279577 req-23d8771a-fef8-43f7-949a-f9cddf0c6648 service nova] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Received event network-vif-deleted-1bd118fa-8a80-4b4f-bf10-f9dc71dae0bb {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 803.513716] env[62499]: DEBUG nova.compute.manager [req-434bbb12-9ab7-4df7-bafb-da88d5f27349 req-c1b9cb46-bea9-46f2-b040-41fef6280e9c service nova] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Received event network-changed-818ca52d-49e7-4e46-a1ac-50d975693883 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 803.513942] env[62499]: DEBUG nova.compute.manager [req-434bbb12-9ab7-4df7-bafb-da88d5f27349 req-c1b9cb46-bea9-46f2-b040-41fef6280e9c service nova] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Refreshing instance network info cache due to event network-changed-818ca52d-49e7-4e46-a1ac-50d975693883. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 803.514266] env[62499]: DEBUG oslo_concurrency.lockutils [req-434bbb12-9ab7-4df7-bafb-da88d5f27349 req-c1b9cb46-bea9-46f2-b040-41fef6280e9c service nova] Acquiring lock "refresh_cache-63b76c39-11ca-4f1e-b336-2caceece1f72" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 803.514478] env[62499]: DEBUG oslo_concurrency.lockutils [req-434bbb12-9ab7-4df7-bafb-da88d5f27349 req-c1b9cb46-bea9-46f2-b040-41fef6280e9c service nova] Acquired lock "refresh_cache-63b76c39-11ca-4f1e-b336-2caceece1f72" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 803.514698] env[62499]: DEBUG nova.network.neutron [req-434bbb12-9ab7-4df7-bafb-da88d5f27349 req-c1b9cb46-bea9-46f2-b040-41fef6280e9c service nova] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Refreshing network info cache for port 818ca52d-49e7-4e46-a1ac-50d975693883 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 803.624184] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Releasing lock "refresh_cache-7bdd4371-b5d5-4053-93b1-75d1c5b9835b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 803.624621] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 803.624886] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 803.625241] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c8f814b7-57d7-4566-9246-9a54757dfa8c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.628300] env[62499]: ERROR nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 818ca52d-49e7-4e46-a1ac-50d975693883, please check neutron logs for more information. [ 803.628300] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 803.628300] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 803.628300] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 803.628300] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 803.628300] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 803.628300] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 803.628300] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 803.628300] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 803.628300] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 803.628300] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 803.628300] env[62499]: ERROR nova.compute.manager raise self.value [ 803.628300] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 803.628300] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 803.628300] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 803.628300] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 803.628671] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 803.628671] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 803.628671] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 818ca52d-49e7-4e46-a1ac-50d975693883, please check neutron logs for more information. [ 803.628671] env[62499]: ERROR nova.compute.manager [ 803.628671] env[62499]: Traceback (most recent call last): [ 803.628671] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 803.628671] env[62499]: listener.cb(fileno) [ 803.628671] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 803.628671] env[62499]: result = function(*args, **kwargs) [ 803.628671] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 803.628671] env[62499]: return func(*args, **kwargs) [ 803.628671] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 803.628671] env[62499]: raise e [ 803.628671] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 803.628671] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 803.628671] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 803.628671] env[62499]: created_port_ids = self._update_ports_for_instance( [ 803.628671] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 803.628671] env[62499]: with excutils.save_and_reraise_exception(): [ 803.628671] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 803.628671] env[62499]: self.force_reraise() [ 803.628671] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 803.628671] env[62499]: raise self.value [ 803.628671] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 803.628671] env[62499]: updated_port = self._update_port( [ 803.628671] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 803.628671] env[62499]: _ensure_no_port_binding_failure(port) [ 803.628671] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 803.628671] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 803.629366] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 818ca52d-49e7-4e46-a1ac-50d975693883, please check neutron logs for more information. [ 803.629366] env[62499]: Removing descriptor: 16 [ 803.629366] env[62499]: ERROR nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 818ca52d-49e7-4e46-a1ac-50d975693883, please check neutron logs for more information. [ 803.629366] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Traceback (most recent call last): [ 803.629366] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 803.629366] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] yield resources [ 803.629366] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 803.629366] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] self.driver.spawn(context, instance, image_meta, [ 803.629366] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 803.629366] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] self._vmops.spawn(context, instance, image_meta, injected_files, [ 803.629366] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 803.629366] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] vm_ref = self.build_virtual_machine(instance, [ 803.629767] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 803.629767] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] vif_infos = vmwarevif.get_vif_info(self._session, [ 803.629767] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 803.629767] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] for vif in network_info: [ 803.629767] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 803.629767] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] return self._sync_wrapper(fn, *args, **kwargs) [ 803.629767] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 803.629767] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] self.wait() [ 803.629767] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 803.629767] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] self[:] = self._gt.wait() [ 803.629767] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 803.629767] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] return self._exit_event.wait() [ 803.629767] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 803.630141] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] result = hub.switch() [ 803.630141] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 803.630141] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] return self.greenlet.switch() [ 803.630141] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 803.630141] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] result = function(*args, **kwargs) [ 803.630141] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 803.630141] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] return func(*args, **kwargs) [ 803.630141] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 803.630141] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] raise e [ 803.630141] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 803.630141] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] nwinfo = self.network_api.allocate_for_instance( [ 803.630141] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 803.630141] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] created_port_ids = self._update_ports_for_instance( [ 803.630512] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 803.630512] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] with excutils.save_and_reraise_exception(): [ 803.630512] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 803.630512] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] self.force_reraise() [ 803.630512] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 803.630512] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] raise self.value [ 803.630512] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 803.630512] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] updated_port = self._update_port( [ 803.630512] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 803.630512] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] _ensure_no_port_binding_failure(port) [ 803.630512] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 803.630512] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] raise exception.PortBindingFailed(port_id=port['id']) [ 803.630880] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] nova.exception.PortBindingFailed: Binding failed for port 818ca52d-49e7-4e46-a1ac-50d975693883, please check neutron logs for more information. [ 803.630880] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] [ 803.630880] env[62499]: INFO nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Terminating instance [ 803.631878] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "refresh_cache-63b76c39-11ca-4f1e-b336-2caceece1f72" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 803.639045] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e12b714-2ece-4328-96ed-c98a8e6e7ca6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.651977] env[62499]: DEBUG oslo_concurrency.lockutils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquiring lock "770160d6-4681-40e0-a812-5c491b804da4" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 803.652243] env[62499]: DEBUG oslo_concurrency.lockutils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "770160d6-4681-40e0-a812-5c491b804da4" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 803.664160] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7bdd4371-b5d5-4053-93b1-75d1c5b9835b could not be found. [ 803.664367] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 803.665027] env[62499]: INFO nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Took 0.04 seconds to destroy the instance on the hypervisor. [ 803.665027] env[62499]: DEBUG oslo.service.loopingcall [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 803.665027] env[62499]: DEBUG nova.compute.manager [-] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 803.665192] env[62499]: DEBUG nova.network.neutron [-] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 803.678880] env[62499]: DEBUG nova.network.neutron [-] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 804.545912] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 3.116s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 804.546422] env[62499]: DEBUG nova.compute.manager [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 804.551354] env[62499]: DEBUG nova.network.neutron [-] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 804.555271] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 28.711s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 804.557691] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Acquiring lock "cbe51201-c036-4f90-9372-8cc2d033f7a2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 804.558065] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Lock "cbe51201-c036-4f90-9372-8cc2d033f7a2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 804.573508] env[62499]: DEBUG nova.network.neutron [req-434bbb12-9ab7-4df7-bafb-da88d5f27349 req-c1b9cb46-bea9-46f2-b040-41fef6280e9c service nova] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 804.681831] env[62499]: DEBUG nova.network.neutron [req-434bbb12-9ab7-4df7-bafb-da88d5f27349 req-c1b9cb46-bea9-46f2-b040-41fef6280e9c service nova] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 805.052069] env[62499]: DEBUG nova.compute.utils [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 805.053154] env[62499]: DEBUG nova.compute.manager [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Not allocating networking since 'none' was specified. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 805.053444] env[62499]: INFO nova.compute.manager [-] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Took 1.39 seconds to deallocate network for instance. [ 805.055399] env[62499]: DEBUG nova.compute.claims [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 805.055569] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 805.187267] env[62499]: DEBUG oslo_concurrency.lockutils [req-434bbb12-9ab7-4df7-bafb-da88d5f27349 req-c1b9cb46-bea9-46f2-b040-41fef6280e9c service nova] Releasing lock "refresh_cache-63b76c39-11ca-4f1e-b336-2caceece1f72" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 805.188058] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquired lock "refresh_cache-63b76c39-11ca-4f1e-b336-2caceece1f72" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 805.188278] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 805.350194] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-878e912d-4aa3-48ef-9cf3-57bba0949928 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 805.357725] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3af42ff-62c3-4f5b-85b1-ad3527fffe4b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 805.388172] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0ba396d-bca5-4820-a4e7-f8d201f1685f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 805.396041] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b35b0e37-1bdf-4719-928c-e126d9171d9e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 805.408992] env[62499]: DEBUG nova.compute.provider_tree [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 805.552360] env[62499]: DEBUG nova.compute.manager [req-d75526a7-aed0-41d1-bc42-6b3c21fdfc4b req-a033dce2-b715-42b1-940a-51d10a2d491a service nova] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Received event network-vif-deleted-818ca52d-49e7-4e46-a1ac-50d975693883 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 805.554230] env[62499]: DEBUG nova.compute.manager [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 805.704358] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 805.780701] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 805.912591] env[62499]: DEBUG nova.scheduler.client.report [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 806.284316] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Releasing lock "refresh_cache-63b76c39-11ca-4f1e-b336-2caceece1f72" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 806.284964] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 806.285312] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 806.285723] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f79f598f-d5e0-42a8-98f4-7e3b548599d9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.295808] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8cf0538b-10df-454a-8365-640e67d3b3fa {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.316305] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 63b76c39-11ca-4f1e-b336-2caceece1f72 could not be found. [ 806.316536] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 806.316722] env[62499]: INFO nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Took 0.03 seconds to destroy the instance on the hypervisor. [ 806.317009] env[62499]: DEBUG oslo.service.loopingcall [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 806.317244] env[62499]: DEBUG nova.compute.manager [-] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 806.317339] env[62499]: DEBUG nova.network.neutron [-] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 806.331756] env[62499]: DEBUG nova.network.neutron [-] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 806.417381] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.862s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 806.418343] env[62499]: ERROR nova.compute.manager [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 7b317662-2319-4c6a-8a87-72b6d90c23aa, please check neutron logs for more information. [ 806.418343] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Traceback (most recent call last): [ 806.418343] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 806.418343] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] self.driver.spawn(context, instance, image_meta, [ 806.418343] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 806.418343] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 806.418343] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 806.418343] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] vm_ref = self.build_virtual_machine(instance, [ 806.418343] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 806.418343] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] vif_infos = vmwarevif.get_vif_info(self._session, [ 806.418343] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 806.418860] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] for vif in network_info: [ 806.418860] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 806.418860] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] return self._sync_wrapper(fn, *args, **kwargs) [ 806.418860] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 806.418860] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] self.wait() [ 806.418860] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 806.418860] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] self[:] = self._gt.wait() [ 806.418860] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 806.418860] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] return self._exit_event.wait() [ 806.418860] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 806.418860] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] current.throw(*self._exc) [ 806.418860] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 806.418860] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] result = function(*args, **kwargs) [ 806.419172] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 806.419172] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] return func(*args, **kwargs) [ 806.419172] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 806.419172] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] raise e [ 806.419172] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 806.419172] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] nwinfo = self.network_api.allocate_for_instance( [ 806.419172] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 806.419172] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] created_port_ids = self._update_ports_for_instance( [ 806.419172] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 806.419172] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] with excutils.save_and_reraise_exception(): [ 806.419172] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 806.419172] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] self.force_reraise() [ 806.419172] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 806.419470] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] raise self.value [ 806.419470] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 806.419470] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] updated_port = self._update_port( [ 806.419470] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 806.419470] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] _ensure_no_port_binding_failure(port) [ 806.419470] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 806.419470] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] raise exception.PortBindingFailed(port_id=port['id']) [ 806.419470] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] nova.exception.PortBindingFailed: Binding failed for port 7b317662-2319-4c6a-8a87-72b6d90c23aa, please check neutron logs for more information. [ 806.419470] env[62499]: ERROR nova.compute.manager [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] [ 806.419470] env[62499]: DEBUG nova.compute.utils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Binding failed for port 7b317662-2319-4c6a-8a87-72b6d90c23aa, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 806.419964] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 28.878s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 806.421755] env[62499]: INFO nova.compute.claims [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 806.425125] env[62499]: DEBUG nova.compute.manager [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Build of instance e7023ea7-9a39-4e5b-90f6-b2499e06ee9d was re-scheduled: Binding failed for port 7b317662-2319-4c6a-8a87-72b6d90c23aa, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 806.425658] env[62499]: DEBUG nova.compute.manager [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 806.425902] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Acquiring lock "refresh_cache-e7023ea7-9a39-4e5b-90f6-b2499e06ee9d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 806.426064] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Acquired lock "refresh_cache-e7023ea7-9a39-4e5b-90f6-b2499e06ee9d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 806.426229] env[62499]: DEBUG nova.network.neutron [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 806.564480] env[62499]: DEBUG nova.compute.manager [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 806.589744] env[62499]: DEBUG nova.virt.hardware [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 806.592028] env[62499]: DEBUG nova.virt.hardware [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 806.592028] env[62499]: DEBUG nova.virt.hardware [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 806.592028] env[62499]: DEBUG nova.virt.hardware [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 806.592028] env[62499]: DEBUG nova.virt.hardware [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 806.592028] env[62499]: DEBUG nova.virt.hardware [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 806.592028] env[62499]: DEBUG nova.virt.hardware [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 806.592277] env[62499]: DEBUG nova.virt.hardware [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 806.592277] env[62499]: DEBUG nova.virt.hardware [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 806.592277] env[62499]: DEBUG nova.virt.hardware [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 806.592277] env[62499]: DEBUG nova.virt.hardware [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 806.593036] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8cd4aeb0-98ba-4a15-b27e-ca2d4efdb7f9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.602606] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2bbba2fd-987a-4b0d-aa7a-80dad8f1b05c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.615684] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Instance VIF info [] {{(pid=62499) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 806.621289] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Creating folder: Project (ef347a1e31844b40b18851d15b81dfe4). Parent ref: group-v285191. {{(pid=62499) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 806.621557] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-ba903571-8983-46f1-93b3-4eb51bdb68e6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.635385] env[62499]: INFO nova.virt.vmwareapi.vm_util [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Created folder: Project (ef347a1e31844b40b18851d15b81dfe4) in parent group-v285191. [ 806.635517] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Creating folder: Instances. Parent ref: group-v285208. {{(pid=62499) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 806.635737] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-c60c25c6-7d9c-40a7-b927-1f950a4c07e5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.645709] env[62499]: INFO nova.virt.vmwareapi.vm_util [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Created folder: Instances in parent group-v285208. [ 806.645934] env[62499]: DEBUG oslo.service.loopingcall [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 806.646125] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Creating VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 806.646316] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-9378ca7b-dfa9-40b4-8be5-d2672dd8c0f9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.661625] env[62499]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 806.661625] env[62499]: value = "task-1335711" [ 806.661625] env[62499]: _type = "Task" [ 806.661625] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 806.668534] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335711, 'name': CreateVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 806.834243] env[62499]: DEBUG nova.network.neutron [-] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 806.943034] env[62499]: DEBUG nova.network.neutron [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 807.022516] env[62499]: DEBUG nova.network.neutron [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 807.171748] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335711, 'name': CreateVM_Task, 'duration_secs': 0.249627} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 807.171748] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Created VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 807.172190] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 807.172398] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquired lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 807.172729] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 807.173655] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-272fd027-6738-4488-86ad-2326b2f6e2ef {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 807.177253] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 807.177253] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52938dcd-dc7a-cee0-469b-498f91477c58" [ 807.177253] env[62499]: _type = "Task" [ 807.177253] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 807.184429] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52938dcd-dc7a-cee0-469b-498f91477c58, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 807.336904] env[62499]: INFO nova.compute.manager [-] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Took 1.02 seconds to deallocate network for instance. [ 807.339493] env[62499]: DEBUG nova.compute.claims [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 807.339746] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 807.524736] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Releasing lock "refresh_cache-e7023ea7-9a39-4e5b-90f6-b2499e06ee9d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 807.524977] env[62499]: DEBUG nova.compute.manager [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 807.525173] env[62499]: DEBUG nova.compute.manager [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 807.525338] env[62499]: DEBUG nova.network.neutron [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 807.538658] env[62499]: DEBUG nova.network.neutron [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 807.688921] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52938dcd-dc7a-cee0-469b-498f91477c58, 'name': SearchDatastore_Task, 'duration_secs': 0.009405} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 807.689272] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Releasing lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 807.689515] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Processing image 0422123c-16ce-4c9d-b1c6-0281fe0e54ad {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 807.689816] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 807.689977] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquired lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 807.690182] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 807.690443] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-11186b14-2436-4891-aafc-23966428ee40 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 807.698531] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 807.698712] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62499) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 807.699506] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-52511a66-cf46-4f71-95f9-1c45e1adf307 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 807.706328] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 807.706328] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]528cb55f-5495-fe5f-ef73-4d56c5145c5a" [ 807.706328] env[62499]: _type = "Task" [ 807.706328] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 807.713805] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]528cb55f-5495-fe5f-ef73-4d56c5145c5a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 807.715208] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e746192-6ea6-44d2-ba06-fceb8e01c7dc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 807.721392] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0430452d-61d3-4c30-aaf1-127c6d2f98b8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 807.753149] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18f5db2e-e2b9-46e2-9aa9-8cf4996d4795 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 807.759932] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0361969f-3ff2-4cf2-bc26-a36ed60fc06c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 807.772851] env[62499]: DEBUG nova.compute.provider_tree [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 808.042054] env[62499]: DEBUG nova.network.neutron [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 808.216774] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]528cb55f-5495-fe5f-ef73-4d56c5145c5a, 'name': SearchDatastore_Task, 'duration_secs': 0.008006} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 808.217573] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-7fdb2959-9e2a-4338-bc52-bc0a79328e79 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 808.222298] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 808.222298] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52dbeeb0-178f-2ad8-94c7-fd813103f284" [ 808.222298] env[62499]: _type = "Task" [ 808.222298] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 808.229320] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52dbeeb0-178f-2ad8-94c7-fd813103f284, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 808.276453] env[62499]: DEBUG nova.scheduler.client.report [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 808.543536] env[62499]: INFO nova.compute.manager [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] [instance: e7023ea7-9a39-4e5b-90f6-b2499e06ee9d] Took 1.02 seconds to deallocate network for instance. [ 808.732178] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52dbeeb0-178f-2ad8-94c7-fd813103f284, 'name': SearchDatastore_Task, 'duration_secs': 0.008298} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 808.732455] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Releasing lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 808.732689] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore2] 008b230e-72b8-43ae-826a-b38111c56e76/008b230e-72b8-43ae-826a-b38111c56e76.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 808.732943] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-8d70b074-e5ba-4797-8fe8-1ff1feb7e3b4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 808.740131] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 808.740131] env[62499]: value = "task-1335712" [ 808.740131] env[62499]: _type = "Task" [ 808.740131] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 808.748945] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335712, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 808.781407] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.361s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 808.782020] env[62499]: DEBUG nova.compute.manager [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 808.785279] env[62499]: DEBUG oslo_concurrency.lockutils [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 29.244s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 808.786702] env[62499]: INFO nova.compute.claims [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 809.249487] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335712, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.458629} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 809.249760] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore2] 008b230e-72b8-43ae-826a-b38111c56e76/008b230e-72b8-43ae-826a-b38111c56e76.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 809.249937] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Extending root virtual disk to 1048576 {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 809.250190] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-c2abe43b-4ba1-43f9-9f25-4173d839f172 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 809.256378] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 809.256378] env[62499]: value = "task-1335713" [ 809.256378] env[62499]: _type = "Task" [ 809.256378] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 809.263254] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335713, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 809.290844] env[62499]: DEBUG nova.compute.utils [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 809.294069] env[62499]: DEBUG nova.compute.manager [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Not allocating networking since 'none' was specified. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 809.575267] env[62499]: INFO nova.scheduler.client.report [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Deleted allocations for instance e7023ea7-9a39-4e5b-90f6-b2499e06ee9d [ 809.766137] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335713, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.060659} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 809.766429] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Extended root virtual disk {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 809.767165] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91752c69-09f6-4934-9f43-f136c32cf11b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 809.786138] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Reconfiguring VM instance instance-00000037 to attach disk [datastore2] 008b230e-72b8-43ae-826a-b38111c56e76/008b230e-72b8-43ae-826a-b38111c56e76.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 809.786413] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-bd5974a7-6a6a-482e-b0c3-a22ad0a8f1b7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 809.801094] env[62499]: DEBUG nova.compute.manager [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 809.810946] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 809.810946] env[62499]: value = "task-1335714" [ 809.810946] env[62499]: _type = "Task" [ 809.810946] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 809.820417] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335714, 'name': ReconfigVM_Task} progress is 5%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 810.083666] env[62499]: DEBUG oslo_concurrency.lockutils [None req-0b950cc2-a421-4531-a439-b37e9bd695fc tempest-ImagesOneServerTestJSON-628175624 tempest-ImagesOneServerTestJSON-628175624-project-member] Lock "e7023ea7-9a39-4e5b-90f6-b2499e06ee9d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 165.283s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 810.212459] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-797c3784-dc32-472c-ad67-05330a62f1fb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.220014] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1df8aff4-3697-417b-8ae5-51e076c1bb05 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.250268] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cbfa8202-5e2f-42f8-a435-45f572c9729c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.257503] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f88a41af-f33c-4af0-8e82-cde0d9990753 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.269992] env[62499]: DEBUG nova.compute.provider_tree [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 810.320604] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335714, 'name': ReconfigVM_Task, 'duration_secs': 0.267729} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 810.320869] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Reconfigured VM instance instance-00000037 to attach disk [datastore2] 008b230e-72b8-43ae-826a-b38111c56e76/008b230e-72b8-43ae-826a-b38111c56e76.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 810.321556] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-7991bc2a-dfdc-411b-9a40-24bcce30355e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.328237] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 810.328237] env[62499]: value = "task-1335715" [ 810.328237] env[62499]: _type = "Task" [ 810.328237] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 810.336300] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335715, 'name': Rename_Task} progress is 5%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 810.586624] env[62499]: DEBUG nova.compute.manager [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 810.772966] env[62499]: DEBUG nova.scheduler.client.report [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 810.813022] env[62499]: DEBUG nova.compute.manager [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 810.841800] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335715, 'name': Rename_Task, 'duration_secs': 0.133201} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 810.844102] env[62499]: DEBUG nova.virt.hardware [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 810.844345] env[62499]: DEBUG nova.virt.hardware [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 810.844496] env[62499]: DEBUG nova.virt.hardware [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 810.844788] env[62499]: DEBUG nova.virt.hardware [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 810.844980] env[62499]: DEBUG nova.virt.hardware [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 810.845173] env[62499]: DEBUG nova.virt.hardware [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 810.845428] env[62499]: DEBUG nova.virt.hardware [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 810.845697] env[62499]: DEBUG nova.virt.hardware [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 810.845911] env[62499]: DEBUG nova.virt.hardware [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 810.846128] env[62499]: DEBUG nova.virt.hardware [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 810.846302] env[62499]: DEBUG nova.virt.hardware [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 810.846595] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Powering on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 810.847379] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9480b1f3-6d10-4f64-af4a-a3994d82e6ac {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.850017] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-b4da29d4-7f18-478d-90ce-8e19a0dfe896 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.860175] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2924af8-5764-4ad7-a227-3764fad9ab64 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.864424] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 810.864424] env[62499]: value = "task-1335716" [ 810.864424] env[62499]: _type = "Task" [ 810.864424] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 810.876572] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Instance VIF info [] {{(pid=62499) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 810.882109] env[62499]: DEBUG oslo.service.loopingcall [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 810.882737] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Creating VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 810.882957] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-a26039ac-5ef3-44f5-a68f-42549db7f4d8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.898837] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335716, 'name': PowerOnVM_Task} progress is 66%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 810.904959] env[62499]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 810.904959] env[62499]: value = "task-1335717" [ 810.904959] env[62499]: _type = "Task" [ 810.904959] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 810.912244] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335717, 'name': CreateVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 811.112715] env[62499]: DEBUG oslo_concurrency.lockutils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 811.279887] env[62499]: DEBUG oslo_concurrency.lockutils [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.494s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 811.280473] env[62499]: DEBUG nova.compute.manager [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 811.283093] env[62499]: DEBUG oslo_concurrency.lockutils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 29.090s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 811.375476] env[62499]: DEBUG oslo_vmware.api [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335716, 'name': PowerOnVM_Task, 'duration_secs': 0.435785} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 811.375806] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Powered on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 811.376152] env[62499]: INFO nova.compute.manager [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Took 4.81 seconds to spawn the instance on the hypervisor. [ 811.376421] env[62499]: DEBUG nova.compute.manager [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 811.377393] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee71a86f-ef66-41e1-a2db-9f9b3b81ebec {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 811.414248] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335717, 'name': CreateVM_Task, 'duration_secs': 0.235737} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 811.414450] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Created VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 811.415029] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 811.415029] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquired lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 811.415324] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 811.415560] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-bdb77131-922e-46ff-858d-c0e63abdd39f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 811.420751] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 811.420751] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52e6137c-026c-d6bf-7838-2fc445569273" [ 811.420751] env[62499]: _type = "Task" [ 811.420751] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 811.428679] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52e6137c-026c-d6bf-7838-2fc445569273, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 811.788134] env[62499]: DEBUG nova.compute.utils [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 811.793365] env[62499]: DEBUG nova.compute.manager [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Not allocating networking since 'none' was specified. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 811.895628] env[62499]: INFO nova.compute.manager [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Took 38.76 seconds to build instance. [ 811.931312] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52e6137c-026c-d6bf-7838-2fc445569273, 'name': SearchDatastore_Task, 'duration_secs': 0.009356} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 811.933709] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Releasing lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 811.933940] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Processing image 0422123c-16ce-4c9d-b1c6-0281fe0e54ad {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 811.934178] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 811.934321] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquired lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 811.934494] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 811.935156] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-11c4e7f1-e98c-43cc-9046-56682f8edb6c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 811.943374] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 811.943550] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=62499) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 811.946387] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-38ece31d-8641-49ca-a4ca-576672a71094 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 811.951772] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 811.951772] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52db5e3f-67d4-ffce-1b4c-c93926f162d3" [ 811.951772] env[62499]: _type = "Task" [ 811.951772] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 811.961295] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52db5e3f-67d4-ffce-1b4c-c93926f162d3, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 812.108076] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7df350fe-9fbf-4542-b30a-b2e1215a035a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.115530] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e508b82c-5906-48d1-ae00-783a07ad64e4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.145875] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ab28ff9-a318-4875-843a-2842d382fbd2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.153662] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c41b219c-2f19-4ca1-bff3-65ca1f223286 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.166755] env[62499]: DEBUG nova.compute.provider_tree [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 812.294211] env[62499]: DEBUG nova.compute.manager [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 812.397324] env[62499]: DEBUG oslo_concurrency.lockutils [None req-fad62c2d-ad48-4f64-a4e9-e8805476c0e6 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "008b230e-72b8-43ae-826a-b38111c56e76" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 143.532s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 812.462285] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52db5e3f-67d4-ffce-1b4c-c93926f162d3, 'name': SearchDatastore_Task, 'duration_secs': 0.011206} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 812.463082] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-88e38a4f-e57e-4d03-854f-7cc662036f87 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.468157] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 812.468157] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52f2710d-0d67-047b-bd72-05b20c6f70f9" [ 812.468157] env[62499]: _type = "Task" [ 812.468157] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 812.475521] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52f2710d-0d67-047b-bd72-05b20c6f70f9, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 812.671047] env[62499]: DEBUG nova.scheduler.client.report [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 812.900260] env[62499]: DEBUG nova.compute.manager [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 812.978289] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52f2710d-0d67-047b-bd72-05b20c6f70f9, 'name': SearchDatastore_Task, 'duration_secs': 0.009357} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 812.978589] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Releasing lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 812.978877] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore1] 39bc8b3a-e9d9-48e3-a193-5fbdc2454346/39bc8b3a-e9d9-48e3-a193-5fbdc2454346.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 812.979166] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-d21bbb9c-3cd6-4fb2-83ce-fd1e1e131364 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.985179] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 812.985179] env[62499]: value = "task-1335718" [ 812.985179] env[62499]: _type = "Task" [ 812.985179] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 812.992309] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335718, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 813.175480] env[62499]: DEBUG oslo_concurrency.lockutils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.892s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 813.176167] env[62499]: ERROR nova.compute.manager [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 0ae0b0d1-4514-4899-a479-3423a0b56ede, please check neutron logs for more information. [ 813.176167] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Traceback (most recent call last): [ 813.176167] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 813.176167] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] self.driver.spawn(context, instance, image_meta, [ 813.176167] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 813.176167] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] self._vmops.spawn(context, instance, image_meta, injected_files, [ 813.176167] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 813.176167] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] vm_ref = self.build_virtual_machine(instance, [ 813.176167] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 813.176167] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] vif_infos = vmwarevif.get_vif_info(self._session, [ 813.176167] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 813.176451] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] for vif in network_info: [ 813.176451] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 813.176451] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] return self._sync_wrapper(fn, *args, **kwargs) [ 813.176451] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 813.176451] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] self.wait() [ 813.176451] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 813.176451] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] self[:] = self._gt.wait() [ 813.176451] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 813.176451] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] return self._exit_event.wait() [ 813.176451] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 813.176451] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] current.throw(*self._exc) [ 813.176451] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 813.176451] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] result = function(*args, **kwargs) [ 813.176801] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 813.176801] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] return func(*args, **kwargs) [ 813.176801] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 813.176801] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] raise e [ 813.176801] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 813.176801] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] nwinfo = self.network_api.allocate_for_instance( [ 813.176801] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 813.176801] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] created_port_ids = self._update_ports_for_instance( [ 813.176801] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 813.176801] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] with excutils.save_and_reraise_exception(): [ 813.176801] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 813.176801] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] self.force_reraise() [ 813.176801] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 813.177101] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] raise self.value [ 813.177101] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 813.177101] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] updated_port = self._update_port( [ 813.177101] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 813.177101] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] _ensure_no_port_binding_failure(port) [ 813.177101] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 813.177101] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] raise exception.PortBindingFailed(port_id=port['id']) [ 813.177101] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] nova.exception.PortBindingFailed: Binding failed for port 0ae0b0d1-4514-4899-a479-3423a0b56ede, please check neutron logs for more information. [ 813.177101] env[62499]: ERROR nova.compute.manager [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] [ 813.177101] env[62499]: DEBUG nova.compute.utils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Binding failed for port 0ae0b0d1-4514-4899-a479-3423a0b56ede, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 813.178233] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.445s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 813.179804] env[62499]: INFO nova.compute.claims [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 813.183040] env[62499]: DEBUG nova.compute.manager [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Build of instance ea14c163-3afe-4a8d-8370-43e26ba7af37 was re-scheduled: Binding failed for port 0ae0b0d1-4514-4899-a479-3423a0b56ede, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 813.183263] env[62499]: DEBUG nova.compute.manager [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 813.183495] env[62499]: DEBUG oslo_concurrency.lockutils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Acquiring lock "refresh_cache-ea14c163-3afe-4a8d-8370-43e26ba7af37" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 813.183642] env[62499]: DEBUG oslo_concurrency.lockutils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Acquired lock "refresh_cache-ea14c163-3afe-4a8d-8370-43e26ba7af37" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 813.183802] env[62499]: DEBUG nova.network.neutron [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 813.305934] env[62499]: DEBUG nova.compute.manager [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 813.333019] env[62499]: DEBUG nova.virt.hardware [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 813.333273] env[62499]: DEBUG nova.virt.hardware [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 813.333484] env[62499]: DEBUG nova.virt.hardware [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 813.333679] env[62499]: DEBUG nova.virt.hardware [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 813.333825] env[62499]: DEBUG nova.virt.hardware [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 813.333976] env[62499]: DEBUG nova.virt.hardware [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 813.334201] env[62499]: DEBUG nova.virt.hardware [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 813.334379] env[62499]: DEBUG nova.virt.hardware [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 813.334578] env[62499]: DEBUG nova.virt.hardware [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 813.334758] env[62499]: DEBUG nova.virt.hardware [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 813.335085] env[62499]: DEBUG nova.virt.hardware [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 813.335863] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4d6e481-38dd-49a3-be39-f1b2e459a952 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.344869] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0069794-1869-4303-90f4-a2a7f19eff7e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.360172] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Instance VIF info [] {{(pid=62499) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 813.366242] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Creating folder: Project (41b5d55537834f248e192c24f5b727f5). Parent ref: group-v285191. {{(pid=62499) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 813.366537] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-ea2d0292-d4d9-4603-814d-ada3a2fcb86a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.379411] env[62499]: INFO nova.virt.vmwareapi.vm_util [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Created folder: Project (41b5d55537834f248e192c24f5b727f5) in parent group-v285191. [ 813.379669] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Creating folder: Instances. Parent ref: group-v285212. {{(pid=62499) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 813.380028] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-6519562f-013d-459c-8c61-db728546635e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.392260] env[62499]: INFO nova.virt.vmwareapi.vm_util [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Created folder: Instances in parent group-v285212. [ 813.392553] env[62499]: DEBUG oslo.service.loopingcall [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 813.392772] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Creating VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 813.393016] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-b18d6e38-9f09-4af4-810b-14875d640c87 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.413627] env[62499]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 813.413627] env[62499]: value = "task-1335721" [ 813.413627] env[62499]: _type = "Task" [ 813.413627] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 813.424959] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335721, 'name': CreateVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 813.427049] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 813.499935] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335718, 'name': CopyVirtualDisk_Task} progress is 77%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 813.711974] env[62499]: DEBUG nova.network.neutron [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 813.808621] env[62499]: DEBUG nova.network.neutron [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 813.924401] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335721, 'name': CreateVM_Task, 'duration_secs': 0.33668} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 813.924699] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Created VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 813.925053] env[62499]: DEBUG oslo_concurrency.lockutils [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 813.925242] env[62499]: DEBUG oslo_concurrency.lockutils [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquired lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 813.925571] env[62499]: DEBUG oslo_concurrency.lockutils [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 813.925838] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-381de695-231a-42f7-811f-d5d5f8c43b8c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.930535] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 813.930535] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52485e06-6e7f-dc8d-e56d-1b0d43d3aa83" [ 813.930535] env[62499]: _type = "Task" [ 813.930535] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 813.939475] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52485e06-6e7f-dc8d-e56d-1b0d43d3aa83, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 813.996503] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335718, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.562971} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 813.996847] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore1] 39bc8b3a-e9d9-48e3-a193-5fbdc2454346/39bc8b3a-e9d9-48e3-a193-5fbdc2454346.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 813.997124] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Extending root virtual disk to 1048576 {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 813.997438] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-54a96652-3d2d-4efc-9a08-1b2c41d7d285 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.003785] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 814.003785] env[62499]: value = "task-1335722" [ 814.003785] env[62499]: _type = "Task" [ 814.003785] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 814.011069] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335722, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 814.315008] env[62499]: DEBUG oslo_concurrency.lockutils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Releasing lock "refresh_cache-ea14c163-3afe-4a8d-8370-43e26ba7af37" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 814.315253] env[62499]: DEBUG nova.compute.manager [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 814.315432] env[62499]: DEBUG nova.compute.manager [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 814.315633] env[62499]: DEBUG nova.network.neutron [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 814.332614] env[62499]: DEBUG nova.network.neutron [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 814.440178] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52485e06-6e7f-dc8d-e56d-1b0d43d3aa83, 'name': SearchDatastore_Task, 'duration_secs': 0.00965} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 814.440394] env[62499]: DEBUG oslo_concurrency.lockutils [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Releasing lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 814.440619] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Processing image 0422123c-16ce-4c9d-b1c6-0281fe0e54ad {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 814.440871] env[62499]: DEBUG oslo_concurrency.lockutils [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 814.441026] env[62499]: DEBUG oslo_concurrency.lockutils [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquired lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 814.441204] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 814.441447] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0a114311-f497-479a-8bbd-30c283b10f59 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.458206] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 814.458396] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=62499) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 814.459119] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-31897ee6-f3cb-44aa-83fd-6c9106a93bc3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.466522] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 814.466522] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]520ff0f4-c053-50fb-6db0-a0a903d17fef" [ 814.466522] env[62499]: _type = "Task" [ 814.466522] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 814.473958] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]520ff0f4-c053-50fb-6db0-a0a903d17fef, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 814.475403] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93a15ba3-9176-4da8-9a95-d861e08c1c8d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.481673] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d335e0f0-e6d4-4ee3-b093-b07b26935284 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.512722] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff152e8d-76d0-4909-97e2-2f596d7fd1d8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.519346] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335722, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.235092} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 814.521289] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Extended root virtual disk {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 814.521992] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2bed6c0-f46c-48fe-a9f8-8c279d82cb6c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.524995] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d31c6d81-79b7-4f45-89ac-f8d7e9c2cff7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.538795] env[62499]: DEBUG nova.compute.provider_tree [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 814.554425] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Reconfiguring VM instance instance-00000038 to attach disk [datastore1] 39bc8b3a-e9d9-48e3-a193-5fbdc2454346/39bc8b3a-e9d9-48e3-a193-5fbdc2454346.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 814.555327] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-e8fa3971-2de4-4092-b7bd-a83f7be20ca6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.574341] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 814.574341] env[62499]: value = "task-1335723" [ 814.574341] env[62499]: _type = "Task" [ 814.574341] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 814.581878] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335723, 'name': ReconfigVM_Task} progress is 5%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 814.835688] env[62499]: DEBUG nova.network.neutron [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 814.977218] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]520ff0f4-c053-50fb-6db0-a0a903d17fef, 'name': SearchDatastore_Task, 'duration_secs': 0.037559} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 814.978064] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-3a6faf91-b143-49d1-9bfa-f63639ca53ee {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.982972] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 814.982972] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52cee056-e33a-a27b-eeb9-c28b0b6884c8" [ 814.982972] env[62499]: _type = "Task" [ 814.982972] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 814.990188] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52cee056-e33a-a27b-eeb9-c28b0b6884c8, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 815.056240] env[62499]: DEBUG nova.scheduler.client.report [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 815.083241] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335723, 'name': ReconfigVM_Task} progress is 14%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 815.338284] env[62499]: INFO nova.compute.manager [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] [instance: ea14c163-3afe-4a8d-8370-43e26ba7af37] Took 1.02 seconds to deallocate network for instance. [ 815.494535] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52cee056-e33a-a27b-eeb9-c28b0b6884c8, 'name': SearchDatastore_Task, 'duration_secs': 0.052596} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 815.494833] env[62499]: DEBUG oslo_concurrency.lockutils [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Releasing lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 815.495163] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore1] 15f8e04f-8c14-4315-8686-da4db517f7b6/15f8e04f-8c14-4315-8686-da4db517f7b6.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 815.495469] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-5a764f8b-b1f0-41e6-8073-eb82343ea80b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 815.502178] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 815.502178] env[62499]: value = "task-1335724" [ 815.502178] env[62499]: _type = "Task" [ 815.502178] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 815.510017] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335724, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 815.561137] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.383s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 815.561647] env[62499]: DEBUG nova.compute.manager [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 815.564497] env[62499]: DEBUG oslo_concurrency.lockutils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.066s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 815.583828] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335723, 'name': ReconfigVM_Task, 'duration_secs': 0.994293} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 815.583985] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Reconfigured VM instance instance-00000038 to attach disk [datastore1] 39bc8b3a-e9d9-48e3-a193-5fbdc2454346/39bc8b3a-e9d9-48e3-a193-5fbdc2454346.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 815.584563] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-c45deb84-5022-4c1d-a952-69fa729b53f7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 815.590700] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 815.590700] env[62499]: value = "task-1335725" [ 815.590700] env[62499]: _type = "Task" [ 815.590700] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 815.600485] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335725, 'name': Rename_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 816.013538] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335724, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.465395} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 816.013800] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore1] 15f8e04f-8c14-4315-8686-da4db517f7b6/15f8e04f-8c14-4315-8686-da4db517f7b6.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 816.014020] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Extending root virtual disk to 1048576 {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 816.014263] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-10c93937-bb59-46df-ad5d-494fda41d204 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.020256] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 816.020256] env[62499]: value = "task-1335726" [ 816.020256] env[62499]: _type = "Task" [ 816.020256] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 816.028187] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335726, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 816.069684] env[62499]: DEBUG nova.compute.utils [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 816.073050] env[62499]: DEBUG nova.compute.manager [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Not allocating networking since 'none' was specified. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 816.101849] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335725, 'name': Rename_Task, 'duration_secs': 0.398297} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 816.102131] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Powering on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 816.102388] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-36b23fbb-cbe2-45ef-b4ba-08d192fc2adf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.110745] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 816.110745] env[62499]: value = "task-1335727" [ 816.110745] env[62499]: _type = "Task" [ 816.110745] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 816.119424] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335727, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 816.377045] env[62499]: INFO nova.scheduler.client.report [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Deleted allocations for instance ea14c163-3afe-4a8d-8370-43e26ba7af37 [ 816.419028] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79d3e08a-59d7-48d5-915a-0dba600ee736 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.427059] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc04307b-8953-4a0c-bd31-7cf351f5c32b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.458262] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6642337-3fb7-4f42-a3f1-8fd192531976 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.466048] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f3d061e-523a-43a1-86b0-95fcf9af9eee {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.480776] env[62499]: DEBUG nova.compute.provider_tree [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 816.530056] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335726, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.055325} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 816.530341] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Extended root virtual disk {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 816.531118] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7d0ae3b-fb43-4d48-b203-349f3b89ec89 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.550865] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Reconfiguring VM instance instance-00000039 to attach disk [datastore1] 15f8e04f-8c14-4315-8686-da4db517f7b6/15f8e04f-8c14-4315-8686-da4db517f7b6.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 816.551020] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-35c31bca-6778-40bc-b27c-adb0603709cc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.570638] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 816.570638] env[62499]: value = "task-1335728" [ 816.570638] env[62499]: _type = "Task" [ 816.570638] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 816.575402] env[62499]: DEBUG nova.compute.manager [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 816.581467] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335728, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 816.622111] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335727, 'name': PowerOnVM_Task} progress is 94%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 816.893608] env[62499]: DEBUG oslo_concurrency.lockutils [None req-254678da-4779-4b4d-b55f-2be552d303d4 tempest-VolumesAdminNegativeTest-61611212 tempest-VolumesAdminNegativeTest-61611212-project-member] Lock "ea14c163-3afe-4a8d-8370-43e26ba7af37" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 169.322s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 816.983538] env[62499]: DEBUG nova.scheduler.client.report [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 817.080298] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335728, 'name': ReconfigVM_Task, 'duration_secs': 0.280435} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 817.080588] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Reconfigured VM instance instance-00000039 to attach disk [datastore1] 15f8e04f-8c14-4315-8686-da4db517f7b6/15f8e04f-8c14-4315-8686-da4db517f7b6.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 817.081198] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-efef6b46-2028-4385-b63f-550360029f94 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 817.090830] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 817.090830] env[62499]: value = "task-1335729" [ 817.090830] env[62499]: _type = "Task" [ 817.090830] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 817.100706] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335729, 'name': Rename_Task} progress is 5%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 817.121635] env[62499]: DEBUG oslo_vmware.api [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335727, 'name': PowerOnVM_Task, 'duration_secs': 0.599465} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 817.121911] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Powered on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 817.122136] env[62499]: INFO nova.compute.manager [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Took 6.31 seconds to spawn the instance on the hypervisor. [ 817.122323] env[62499]: DEBUG nova.compute.manager [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 817.123127] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae663340-9ca9-409f-8778-650425fdd2a1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 817.398992] env[62499]: DEBUG nova.compute.manager [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 817.489021] env[62499]: DEBUG oslo_concurrency.lockutils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.924s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 817.489370] env[62499]: ERROR nova.compute.manager [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 2e2b4e94-b7e3-44ca-85a2-06de83bce3e0, please check neutron logs for more information. [ 817.489370] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Traceback (most recent call last): [ 817.489370] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 817.489370] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] self.driver.spawn(context, instance, image_meta, [ 817.489370] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 817.489370] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 817.489370] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 817.489370] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] vm_ref = self.build_virtual_machine(instance, [ 817.489370] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 817.489370] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] vif_infos = vmwarevif.get_vif_info(self._session, [ 817.489370] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 817.489754] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] for vif in network_info: [ 817.489754] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 817.489754] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] return self._sync_wrapper(fn, *args, **kwargs) [ 817.489754] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 817.489754] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] self.wait() [ 817.489754] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 817.489754] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] self[:] = self._gt.wait() [ 817.489754] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 817.489754] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] return self._exit_event.wait() [ 817.489754] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 817.489754] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] result = hub.switch() [ 817.489754] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 817.489754] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] return self.greenlet.switch() [ 817.490087] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 817.490087] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] result = function(*args, **kwargs) [ 817.490087] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 817.490087] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] return func(*args, **kwargs) [ 817.490087] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 817.490087] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] raise e [ 817.490087] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 817.490087] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] nwinfo = self.network_api.allocate_for_instance( [ 817.490087] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 817.490087] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] created_port_ids = self._update_ports_for_instance( [ 817.490087] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 817.490087] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] with excutils.save_and_reraise_exception(): [ 817.490087] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 817.490369] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] self.force_reraise() [ 817.490369] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 817.490369] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] raise self.value [ 817.490369] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 817.490369] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] updated_port = self._update_port( [ 817.490369] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 817.490369] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] _ensure_no_port_binding_failure(port) [ 817.490369] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 817.490369] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] raise exception.PortBindingFailed(port_id=port['id']) [ 817.490369] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] nova.exception.PortBindingFailed: Binding failed for port 2e2b4e94-b7e3-44ca-85a2-06de83bce3e0, please check neutron logs for more information. [ 817.490369] env[62499]: ERROR nova.compute.manager [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] [ 817.490621] env[62499]: DEBUG nova.compute.utils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Binding failed for port 2e2b4e94-b7e3-44ca-85a2-06de83bce3e0, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 817.492875] env[62499]: DEBUG oslo_concurrency.lockutils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.395s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 817.494444] env[62499]: INFO nova.compute.claims [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 817.495800] env[62499]: DEBUG nova.compute.manager [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Build of instance 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5 was re-scheduled: Binding failed for port 2e2b4e94-b7e3-44ca-85a2-06de83bce3e0, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 817.496242] env[62499]: DEBUG nova.compute.manager [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 817.496462] env[62499]: DEBUG oslo_concurrency.lockutils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Acquiring lock "refresh_cache-6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 817.496613] env[62499]: DEBUG oslo_concurrency.lockutils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Acquired lock "refresh_cache-6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 817.496763] env[62499]: DEBUG nova.network.neutron [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 817.588609] env[62499]: DEBUG nova.compute.manager [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 817.600674] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335729, 'name': Rename_Task, 'duration_secs': 0.135322} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 817.600965] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Powering on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 817.601680] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-b36d06fc-27aa-4a85-8858-9380b9409786 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 817.608909] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 817.608909] env[62499]: value = "task-1335730" [ 817.608909] env[62499]: _type = "Task" [ 817.608909] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 817.619154] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335730, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 817.622203] env[62499]: DEBUG nova.virt.hardware [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 817.622454] env[62499]: DEBUG nova.virt.hardware [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 817.622642] env[62499]: DEBUG nova.virt.hardware [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 817.622832] env[62499]: DEBUG nova.virt.hardware [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 817.622980] env[62499]: DEBUG nova.virt.hardware [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 817.623142] env[62499]: DEBUG nova.virt.hardware [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 817.623346] env[62499]: DEBUG nova.virt.hardware [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 817.623506] env[62499]: DEBUG nova.virt.hardware [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 817.623669] env[62499]: DEBUG nova.virt.hardware [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 817.623831] env[62499]: DEBUG nova.virt.hardware [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 817.624058] env[62499]: DEBUG nova.virt.hardware [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 817.624811] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d57f914-cb7a-4197-9ccf-1f60d303d1c9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 817.632198] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7028d0c5-1bda-42d9-96e8-4c34edcce966 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 817.652241] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Instance VIF info [] {{(pid=62499) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 817.657883] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Creating folder: Project (5bea0056ac8c4a36b8979705ef763b02). Parent ref: group-v285191. {{(pid=62499) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 817.658512] env[62499]: INFO nova.compute.manager [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Took 40.13 seconds to build instance. [ 817.659440] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-04a76414-1f19-483a-8ae3-85f45e13d9ad {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 817.669483] env[62499]: INFO nova.virt.vmwareapi.vm_util [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Created folder: Project (5bea0056ac8c4a36b8979705ef763b02) in parent group-v285191. [ 817.669682] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Creating folder: Instances. Parent ref: group-v285215. {{(pid=62499) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 817.669947] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-2ba1a3f8-fb60-4d87-8b56-cbba818ddfab {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 817.680784] env[62499]: INFO nova.virt.vmwareapi.vm_util [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Created folder: Instances in parent group-v285215. [ 817.681053] env[62499]: DEBUG oslo.service.loopingcall [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 817.681250] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Creating VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 817.681456] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-3f3157f0-1f17-4828-9b39-876bc43a6dfb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 817.698791] env[62499]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 817.698791] env[62499]: value = "task-1335733" [ 817.698791] env[62499]: _type = "Task" [ 817.698791] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 817.706402] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335733, 'name': CreateVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 817.924391] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 818.016305] env[62499]: DEBUG nova.network.neutron [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 818.077966] env[62499]: DEBUG nova.network.neutron [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 818.120019] env[62499]: DEBUG oslo_vmware.api [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335730, 'name': PowerOnVM_Task, 'duration_secs': 0.462162} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 818.120019] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Powered on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 818.120019] env[62499]: INFO nova.compute.manager [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Took 4.81 seconds to spawn the instance on the hypervisor. [ 818.120019] env[62499]: DEBUG nova.compute.manager [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 818.120532] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-660b9aff-fbb4-400a-a719-ffc226f4c073 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.162846] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4d1b4e6e-33f6-43d4-b753-8d51290553bf tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "39bc8b3a-e9d9-48e3-a193-5fbdc2454346" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 149.018s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 818.211444] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335733, 'name': CreateVM_Task, 'duration_secs': 0.324968} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 818.211775] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Created VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 818.212483] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 818.213701] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Acquired lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 818.213701] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 818.213701] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-1e05036e-c96b-4315-91c3-6f0fcec57f05 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.217850] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Waiting for the task: (returnval){ [ 818.217850] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52c6de70-3688-8d26-a820-a99e7361dd1a" [ 818.217850] env[62499]: _type = "Task" [ 818.217850] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 818.225340] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52c6de70-3688-8d26-a820-a99e7361dd1a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 818.281731] env[62499]: INFO nova.compute.manager [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Rebuilding instance [ 818.325025] env[62499]: DEBUG nova.compute.manager [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 818.325868] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78bee7f9-1126-457e-88cf-10547d0ac280 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.582516] env[62499]: DEBUG oslo_concurrency.lockutils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Releasing lock "refresh_cache-6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 818.582516] env[62499]: DEBUG nova.compute.manager [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 818.582516] env[62499]: DEBUG nova.compute.manager [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 818.582516] env[62499]: DEBUG nova.network.neutron [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 818.599763] env[62499]: DEBUG nova.network.neutron [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 818.641471] env[62499]: INFO nova.compute.manager [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Took 39.12 seconds to build instance. [ 818.665954] env[62499]: DEBUG nova.compute.manager [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 818.739698] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52c6de70-3688-8d26-a820-a99e7361dd1a, 'name': SearchDatastore_Task, 'duration_secs': 0.035178} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 818.740027] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Releasing lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 818.740380] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Processing image 0422123c-16ce-4c9d-b1c6-0281fe0e54ad {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 818.740632] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 818.742634] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Acquired lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 818.742634] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 818.742634] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-b100ec6c-cbe2-4124-bd55-f38e3fd31204 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.753202] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 818.753393] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62499) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 818.754123] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-6b2fdc1c-a447-4cd4-98e3-e0fa0f9104b1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.761823] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Waiting for the task: (returnval){ [ 818.761823] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52178583-cb7f-63a8-f2f9-f76b8b68899b" [ 818.761823] env[62499]: _type = "Task" [ 818.761823] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 818.770131] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52178583-cb7f-63a8-f2f9-f76b8b68899b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 818.837680] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Powering off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 818.841204] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-74c71be5-9473-4091-8a8a-c94cd05fe643 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.849896] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 818.849896] env[62499]: value = "task-1335734" [ 818.849896] env[62499]: _type = "Task" [ 818.849896] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 818.859033] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335734, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 818.908875] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5843df0-b1f4-4527-9128-55c3f3504440 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.918225] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c117a862-202e-4cc7-9603-e5dfb33fc243 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.954599] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da975aff-c58c-43c3-a916-31307f7a880f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.967250] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-03d5c0b7-3307-46ab-8dec-00dc3615e3dd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.987963] env[62499]: DEBUG nova.compute.provider_tree [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 819.103050] env[62499]: DEBUG nova.network.neutron [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 819.143619] env[62499]: DEBUG oslo_concurrency.lockutils [None req-290bd42d-eb9d-4e1c-a1ae-5677ee905ec6 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Lock "15f8e04f-8c14-4315-8686-da4db517f7b6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 149.226s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 819.192459] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 819.272511] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52178583-cb7f-63a8-f2f9-f76b8b68899b, 'name': SearchDatastore_Task, 'duration_secs': 0.021046} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 819.273397] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-211cf501-7127-4f95-af09-b3f51836029a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 819.279833] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Waiting for the task: (returnval){ [ 819.279833] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52964e0a-d2f2-0081-dd9a-23b9bec1828e" [ 819.279833] env[62499]: _type = "Task" [ 819.279833] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 819.286322] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52964e0a-d2f2-0081-dd9a-23b9bec1828e, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 819.360739] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335734, 'name': PowerOffVM_Task, 'duration_secs': 0.199767} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 819.361088] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Powered off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 819.361239] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 819.362113] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c5c474f-1883-435b-9a16-4f11cd6dfeb6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 819.368550] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Unregistering the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 819.368623] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-60d11de3-9f2e-4291-bfd5-c36f58f20937 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 819.390174] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Unregistered the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 819.390372] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Deleting contents of the VM from datastore datastore1 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 819.390566] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Deleting the datastore file [datastore1] 39bc8b3a-e9d9-48e3-a193-5fbdc2454346 {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 819.390789] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-601fc97f-db85-4f50-bf8a-f9388f1a1ac3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 819.396478] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 819.396478] env[62499]: value = "task-1335736" [ 819.396478] env[62499]: _type = "Task" [ 819.396478] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 819.404472] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335736, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 819.491504] env[62499]: DEBUG nova.scheduler.client.report [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 819.606825] env[62499]: INFO nova.compute.manager [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] [instance: 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5] Took 1.02 seconds to deallocate network for instance. [ 819.645257] env[62499]: DEBUG nova.compute.manager [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 819.790432] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52964e0a-d2f2-0081-dd9a-23b9bec1828e, 'name': SearchDatastore_Task, 'duration_secs': 0.008578} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 819.790687] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Releasing lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 819.790968] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore2] 57d21761-431b-4a41-86eb-038f8c35d8ef/57d21761-431b-4a41-86eb-038f8c35d8ef.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 819.791229] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-8493e0b5-07ab-499f-900b-701f7704ef10 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 819.797419] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Waiting for the task: (returnval){ [ 819.797419] env[62499]: value = "task-1335737" [ 819.797419] env[62499]: _type = "Task" [ 819.797419] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 819.804699] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335737, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 819.906739] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335736, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.087239} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 819.906975] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Deleted the datastore file {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 819.907190] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Deleted contents of the VM from datastore datastore1 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 819.907416] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 819.952819] env[62499]: INFO nova.compute.manager [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Rebuilding instance [ 819.997326] env[62499]: DEBUG oslo_concurrency.lockutils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.505s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 819.997482] env[62499]: DEBUG nova.compute.manager [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 820.000686] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.945s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.015834] env[62499]: DEBUG nova.compute.manager [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 820.018641] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8da77363-67ce-485d-a972-4e3fe6ac73d2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 820.166449] env[62499]: DEBUG oslo_concurrency.lockutils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.307435] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335737, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.474265} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 820.307722] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore2] 57d21761-431b-4a41-86eb-038f8c35d8ef/57d21761-431b-4a41-86eb-038f8c35d8ef.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 820.307947] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Extending root virtual disk to 1048576 {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 820.308212] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-f7f08122-d0b6-4330-a77f-ce882120fdcb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 820.315677] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Waiting for the task: (returnval){ [ 820.315677] env[62499]: value = "task-1335738" [ 820.315677] env[62499]: _type = "Task" [ 820.315677] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 820.323676] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335738, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 820.506682] env[62499]: DEBUG nova.compute.utils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 820.512102] env[62499]: DEBUG nova.compute.manager [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 820.512749] env[62499]: DEBUG nova.network.neutron [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 820.533255] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Powering off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 820.533801] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-b35aadf6-adfe-40a0-9e03-6357c5733715 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 820.542152] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 820.542152] env[62499]: value = "task-1335739" [ 820.542152] env[62499]: _type = "Task" [ 820.542152] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 820.554077] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335739, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 820.583892] env[62499]: DEBUG nova.policy [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2fedbea4e5b945918a1e906af369211f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '4f1e8a17916f4b179a86748cb34ec51d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 820.639287] env[62499]: INFO nova.scheduler.client.report [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Deleted allocations for instance 6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5 [ 820.828981] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335738, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.066744} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 820.829326] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Extended root virtual disk {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 820.830278] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9b8fe894-02f8-492e-af44-dc406f1c7068 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 820.851846] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Reconfiguring VM instance instance-0000003a to attach disk [datastore2] 57d21761-431b-4a41-86eb-038f8c35d8ef/57d21761-431b-4a41-86eb-038f8c35d8ef.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 820.854693] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-8288c363-5a9d-4867-97d1-82e984784d0d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 820.876688] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Waiting for the task: (returnval){ [ 820.876688] env[62499]: value = "task-1335740" [ 820.876688] env[62499]: _type = "Task" [ 820.876688] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 820.887379] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335740, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 820.921156] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f22959c-0937-4bfa-8378-26a1c6b5f552 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 820.929444] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2714e37a-fffd-49bd-b67b-021f3629d2fa {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 820.969967] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e3a2015-4409-4bc8-8dd8-2de0a702bbab {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 820.978174] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f95ce18-ce57-434d-80b3-3d12b0c449f5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 820.984842] env[62499]: DEBUG nova.virt.hardware [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 820.985154] env[62499]: DEBUG nova.virt.hardware [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 820.985240] env[62499]: DEBUG nova.virt.hardware [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 820.985441] env[62499]: DEBUG nova.virt.hardware [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 820.985613] env[62499]: DEBUG nova.virt.hardware [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 820.985848] env[62499]: DEBUG nova.virt.hardware [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 820.985971] env[62499]: DEBUG nova.virt.hardware [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 820.986189] env[62499]: DEBUG nova.virt.hardware [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 820.986392] env[62499]: DEBUG nova.virt.hardware [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 820.986553] env[62499]: DEBUG nova.virt.hardware [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 820.986806] env[62499]: DEBUG nova.virt.hardware [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 820.987572] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e147440-29fa-45ce-8f10-3ea603666995 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.001152] env[62499]: DEBUG nova.compute.provider_tree [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 821.006362] env[62499]: DEBUG nova.network.neutron [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Successfully created port: 1863af9b-2409-4da1-a6dc-d2bb4324e5ec {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 821.009464] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14597e2f-88f2-4d1c-ac0d-20bea637c860 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.014939] env[62499]: DEBUG nova.compute.manager [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 821.030493] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Instance VIF info [] {{(pid=62499) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 821.036373] env[62499]: DEBUG oslo.service.loopingcall [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 821.036894] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Creating VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 821.037131] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-fcf3dd2a-d618-46bb-9069-6fb76ad87e2c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.059149] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335739, 'name': PowerOffVM_Task, 'duration_secs': 0.182367} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 821.060420] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Powered off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 821.060584] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 821.060851] env[62499]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 821.060851] env[62499]: value = "task-1335741" [ 821.060851] env[62499]: _type = "Task" [ 821.060851] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 821.061572] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f2155c7-f6b6-412b-bd43-bcd63c7d5908 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.071512] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Unregistering the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 821.075347] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-99af2b8d-53cf-4576-b4d6-63ebd767b59a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.076652] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335741, 'name': CreateVM_Task} progress is 6%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 821.103928] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Unregistered the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 821.104144] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Deleting contents of the VM from datastore datastore1 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 821.104225] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Deleting the datastore file [datastore1] 15f8e04f-8c14-4315-8686-da4db517f7b6 {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 821.104497] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-6100684f-2673-4e06-af2c-d50ccdd6b39a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.111604] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 821.111604] env[62499]: value = "task-1335743" [ 821.111604] env[62499]: _type = "Task" [ 821.111604] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 821.120056] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335743, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 821.149473] env[62499]: DEBUG oslo_concurrency.lockutils [None req-cc2a4911-53bc-4563-aae5-9bb09073f3f0 tempest-TenantUsagesTestJSON-463017862 tempest-TenantUsagesTestJSON-463017862-project-member] Lock "6fef0de4-89d3-4acb-bfa0-9f67eca9c3f5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 162.738s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 821.386906] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335740, 'name': ReconfigVM_Task, 'duration_secs': 0.291001} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 821.386906] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Reconfigured VM instance instance-0000003a to attach disk [datastore2] 57d21761-431b-4a41-86eb-038f8c35d8ef/57d21761-431b-4a41-86eb-038f8c35d8ef.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 821.388011] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-7224cc1c-5166-4625-a915-0b4b4ef02c0f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.393561] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Waiting for the task: (returnval){ [ 821.393561] env[62499]: value = "task-1335744" [ 821.393561] env[62499]: _type = "Task" [ 821.393561] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 821.405718] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335744, 'name': Rename_Task} progress is 6%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 821.510144] env[62499]: DEBUG nova.scheduler.client.report [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 821.574792] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335741, 'name': CreateVM_Task, 'duration_secs': 0.49857} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 821.574993] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Created VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 821.575384] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 821.575537] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquired lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 821.575846] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 821.576101] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-092fea01-30c8-4681-9d42-45c7cc34c711 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.580586] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 821.580586] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]527a5f31-9962-5ac8-27c2-9af9149f2985" [ 821.580586] env[62499]: _type = "Task" [ 821.580586] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 821.590603] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]527a5f31-9962-5ac8-27c2-9af9149f2985, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 821.623045] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335743, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.12644} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 821.623320] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Deleted the datastore file {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 821.623505] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Deleted contents of the VM from datastore datastore1 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 821.623680] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 821.654172] env[62499]: DEBUG nova.compute.manager [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 821.868431] env[62499]: DEBUG nova.compute.manager [req-e6409fd5-f369-4195-b07c-ab7909adf445 req-1001c65f-c7f1-4efe-9be0-8a89c5f44a54 service nova] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Received event network-changed-1863af9b-2409-4da1-a6dc-d2bb4324e5ec {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 821.868650] env[62499]: DEBUG nova.compute.manager [req-e6409fd5-f369-4195-b07c-ab7909adf445 req-1001c65f-c7f1-4efe-9be0-8a89c5f44a54 service nova] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Refreshing instance network info cache due to event network-changed-1863af9b-2409-4da1-a6dc-d2bb4324e5ec. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 821.868874] env[62499]: DEBUG oslo_concurrency.lockutils [req-e6409fd5-f369-4195-b07c-ab7909adf445 req-1001c65f-c7f1-4efe-9be0-8a89c5f44a54 service nova] Acquiring lock "refresh_cache-261cbefa-fb2d-48da-a4fe-80b744a931f6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 821.869030] env[62499]: DEBUG oslo_concurrency.lockutils [req-e6409fd5-f369-4195-b07c-ab7909adf445 req-1001c65f-c7f1-4efe-9be0-8a89c5f44a54 service nova] Acquired lock "refresh_cache-261cbefa-fb2d-48da-a4fe-80b744a931f6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 821.869285] env[62499]: DEBUG nova.network.neutron [req-e6409fd5-f369-4195-b07c-ab7909adf445 req-1001c65f-c7f1-4efe-9be0-8a89c5f44a54 service nova] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Refreshing network info cache for port 1863af9b-2409-4da1-a6dc-d2bb4324e5ec {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 821.905998] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335744, 'name': Rename_Task} progress is 99%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 822.016345] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.016s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 822.017121] env[62499]: ERROR nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1bd118fa-8a80-4b4f-bf10-f9dc71dae0bb, please check neutron logs for more information. [ 822.017121] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Traceback (most recent call last): [ 822.017121] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 822.017121] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] self.driver.spawn(context, instance, image_meta, [ 822.017121] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 822.017121] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 822.017121] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 822.017121] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] vm_ref = self.build_virtual_machine(instance, [ 822.017121] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 822.017121] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] vif_infos = vmwarevif.get_vif_info(self._session, [ 822.017121] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 822.017664] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] for vif in network_info: [ 822.017664] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 822.017664] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] return self._sync_wrapper(fn, *args, **kwargs) [ 822.017664] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 822.017664] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] self.wait() [ 822.017664] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 822.017664] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] self[:] = self._gt.wait() [ 822.017664] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 822.017664] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] return self._exit_event.wait() [ 822.017664] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 822.017664] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] result = hub.switch() [ 822.017664] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 822.017664] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] return self.greenlet.switch() [ 822.018252] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 822.018252] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] result = function(*args, **kwargs) [ 822.018252] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 822.018252] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] return func(*args, **kwargs) [ 822.018252] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 822.018252] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] raise e [ 822.018252] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 822.018252] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] nwinfo = self.network_api.allocate_for_instance( [ 822.018252] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 822.018252] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] created_port_ids = self._update_ports_for_instance( [ 822.018252] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 822.018252] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] with excutils.save_and_reraise_exception(): [ 822.018252] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 822.018810] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] self.force_reraise() [ 822.018810] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 822.018810] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] raise self.value [ 822.018810] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 822.018810] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] updated_port = self._update_port( [ 822.018810] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 822.018810] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] _ensure_no_port_binding_failure(port) [ 822.018810] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 822.018810] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] raise exception.PortBindingFailed(port_id=port['id']) [ 822.018810] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] nova.exception.PortBindingFailed: Binding failed for port 1bd118fa-8a80-4b4f-bf10-f9dc71dae0bb, please check neutron logs for more information. [ 822.018810] env[62499]: ERROR nova.compute.manager [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] [ 822.019311] env[62499]: DEBUG nova.compute.utils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Binding failed for port 1bd118fa-8a80-4b4f-bf10-f9dc71dae0bb, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 822.019672] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.680s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 822.023969] env[62499]: DEBUG nova.compute.manager [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 822.026426] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Build of instance 7bdd4371-b5d5-4053-93b1-75d1c5b9835b was re-scheduled: Binding failed for port 1bd118fa-8a80-4b4f-bf10-f9dc71dae0bb, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 822.026859] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 822.027299] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "refresh_cache-7bdd4371-b5d5-4053-93b1-75d1c5b9835b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 822.027299] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquired lock "refresh_cache-7bdd4371-b5d5-4053-93b1-75d1c5b9835b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 822.027428] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 822.056525] env[62499]: DEBUG nova.virt.hardware [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 822.057601] env[62499]: DEBUG nova.virt.hardware [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 822.057601] env[62499]: DEBUG nova.virt.hardware [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 822.057601] env[62499]: DEBUG nova.virt.hardware [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 822.057601] env[62499]: DEBUG nova.virt.hardware [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 822.058109] env[62499]: DEBUG nova.virt.hardware [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 822.058460] env[62499]: DEBUG nova.virt.hardware [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 822.058679] env[62499]: DEBUG nova.virt.hardware [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 822.058905] env[62499]: DEBUG nova.virt.hardware [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 822.059372] env[62499]: DEBUG nova.virt.hardware [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 822.059787] env[62499]: DEBUG nova.virt.hardware [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 822.061627] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea197859-9abe-4af5-b921-cbe142275f17 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.065737] env[62499]: ERROR nova.compute.manager [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1863af9b-2409-4da1-a6dc-d2bb4324e5ec, please check neutron logs for more information. [ 822.065737] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 822.065737] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 822.065737] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 822.065737] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 822.065737] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 822.065737] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 822.065737] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 822.065737] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 822.065737] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 822.065737] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 822.065737] env[62499]: ERROR nova.compute.manager raise self.value [ 822.065737] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 822.065737] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 822.065737] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 822.065737] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 822.066184] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 822.066184] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 822.066184] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1863af9b-2409-4da1-a6dc-d2bb4324e5ec, please check neutron logs for more information. [ 822.066184] env[62499]: ERROR nova.compute.manager [ 822.066184] env[62499]: Traceback (most recent call last): [ 822.066184] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 822.066184] env[62499]: listener.cb(fileno) [ 822.066184] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 822.066184] env[62499]: result = function(*args, **kwargs) [ 822.066184] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 822.066184] env[62499]: return func(*args, **kwargs) [ 822.066184] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 822.066184] env[62499]: raise e [ 822.066184] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 822.066184] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 822.066184] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 822.066184] env[62499]: created_port_ids = self._update_ports_for_instance( [ 822.066184] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 822.066184] env[62499]: with excutils.save_and_reraise_exception(): [ 822.066184] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 822.066184] env[62499]: self.force_reraise() [ 822.066184] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 822.066184] env[62499]: raise self.value [ 822.066184] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 822.066184] env[62499]: updated_port = self._update_port( [ 822.066184] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 822.066184] env[62499]: _ensure_no_port_binding_failure(port) [ 822.066184] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 822.066184] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 822.066882] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 1863af9b-2409-4da1-a6dc-d2bb4324e5ec, please check neutron logs for more information. [ 822.066882] env[62499]: Removing descriptor: 16 [ 822.072571] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-727a7a59-20ad-449d-bfa1-8d2ad78627ac {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.086818] env[62499]: ERROR nova.compute.manager [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1863af9b-2409-4da1-a6dc-d2bb4324e5ec, please check neutron logs for more information. [ 822.086818] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Traceback (most recent call last): [ 822.086818] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 822.086818] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] yield resources [ 822.086818] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 822.086818] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] self.driver.spawn(context, instance, image_meta, [ 822.086818] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 822.086818] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 822.086818] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 822.086818] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] vm_ref = self.build_virtual_machine(instance, [ 822.086818] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 822.087104] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] vif_infos = vmwarevif.get_vif_info(self._session, [ 822.087104] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 822.087104] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] for vif in network_info: [ 822.087104] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 822.087104] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] return self._sync_wrapper(fn, *args, **kwargs) [ 822.087104] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 822.087104] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] self.wait() [ 822.087104] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 822.087104] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] self[:] = self._gt.wait() [ 822.087104] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 822.087104] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] return self._exit_event.wait() [ 822.087104] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 822.087104] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] current.throw(*self._exc) [ 822.087396] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 822.087396] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] result = function(*args, **kwargs) [ 822.087396] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 822.087396] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] return func(*args, **kwargs) [ 822.087396] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 822.087396] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] raise e [ 822.087396] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 822.087396] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] nwinfo = self.network_api.allocate_for_instance( [ 822.087396] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 822.087396] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] created_port_ids = self._update_ports_for_instance( [ 822.087396] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 822.087396] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] with excutils.save_and_reraise_exception(): [ 822.087396] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 822.087708] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] self.force_reraise() [ 822.087708] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 822.087708] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] raise self.value [ 822.087708] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 822.087708] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] updated_port = self._update_port( [ 822.087708] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 822.087708] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] _ensure_no_port_binding_failure(port) [ 822.087708] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 822.087708] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] raise exception.PortBindingFailed(port_id=port['id']) [ 822.087708] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] nova.exception.PortBindingFailed: Binding failed for port 1863af9b-2409-4da1-a6dc-d2bb4324e5ec, please check neutron logs for more information. [ 822.087708] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] [ 822.087708] env[62499]: INFO nova.compute.manager [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Terminating instance [ 822.088994] env[62499]: DEBUG oslo_concurrency.lockutils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Acquiring lock "refresh_cache-261cbefa-fb2d-48da-a4fe-80b744a931f6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 822.097194] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]527a5f31-9962-5ac8-27c2-9af9149f2985, 'name': SearchDatastore_Task, 'duration_secs': 0.015365} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 822.097460] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Releasing lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 822.097672] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Processing image 0422123c-16ce-4c9d-b1c6-0281fe0e54ad {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 822.097896] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 822.098084] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquired lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 822.098275] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 822.098513] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-8feab9ef-c185-423a-9204-e15d431c2d09 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.106312] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 822.106493] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62499) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 822.107170] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c9a30d83-f753-469f-a8ae-92c4f28eac84 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.111855] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 822.111855] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]522d998b-b214-6dbc-784e-23f23394301a" [ 822.111855] env[62499]: _type = "Task" [ 822.111855] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 822.119375] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]522d998b-b214-6dbc-784e-23f23394301a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 822.180885] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 822.389996] env[62499]: DEBUG nova.network.neutron [req-e6409fd5-f369-4195-b07c-ab7909adf445 req-1001c65f-c7f1-4efe-9be0-8a89c5f44a54 service nova] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 822.404524] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335744, 'name': Rename_Task} progress is 99%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 822.478543] env[62499]: DEBUG nova.network.neutron [req-e6409fd5-f369-4195-b07c-ab7909adf445 req-1001c65f-c7f1-4efe-9be0-8a89c5f44a54 service nova] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 822.550287] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 822.623458] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]522d998b-b214-6dbc-784e-23f23394301a, 'name': SearchDatastore_Task, 'duration_secs': 0.008411} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 822.626283] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-d7d2d05f-6d02-403d-b93f-4635a3f28520 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.631338] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 822.631338] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]5213bbfa-00e7-f20c-77db-b649065f7610" [ 822.631338] env[62499]: _type = "Task" [ 822.631338] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 822.649758] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]5213bbfa-00e7-f20c-77db-b649065f7610, 'name': SearchDatastore_Task, 'duration_secs': 0.009277} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 822.652227] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Releasing lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 822.652553] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore2] 39bc8b3a-e9d9-48e3-a193-5fbdc2454346/39bc8b3a-e9d9-48e3-a193-5fbdc2454346.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 822.653426] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 822.654506] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-4a8274f9-8b71-4283-b46a-efb2c62bc2ce {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.665163] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 822.665163] env[62499]: value = "task-1335745" [ 822.665163] env[62499]: _type = "Task" [ 822.665163] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 822.673010] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335745, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 822.675067] env[62499]: DEBUG nova.virt.hardware [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 822.675296] env[62499]: DEBUG nova.virt.hardware [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 822.675451] env[62499]: DEBUG nova.virt.hardware [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 822.675630] env[62499]: DEBUG nova.virt.hardware [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 822.675773] env[62499]: DEBUG nova.virt.hardware [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 822.675927] env[62499]: DEBUG nova.virt.hardware [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 822.676122] env[62499]: DEBUG nova.virt.hardware [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 822.676280] env[62499]: DEBUG nova.virt.hardware [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 822.676442] env[62499]: DEBUG nova.virt.hardware [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 822.676599] env[62499]: DEBUG nova.virt.hardware [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 822.676766] env[62499]: DEBUG nova.virt.hardware [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 822.680075] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd63074f-94c6-4698-9075-1a30cb37f4ad {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.687427] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0e9a9d1-ce27-42eb-87ea-10d000f0bc9d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.703456] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Instance VIF info [] {{(pid=62499) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 822.708583] env[62499]: DEBUG oslo.service.loopingcall [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 822.712020] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Creating VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 822.712020] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-37a4323b-5c84-4459-8efb-fa0210d79000 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.729187] env[62499]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 822.729187] env[62499]: value = "task-1335746" [ 822.729187] env[62499]: _type = "Task" [ 822.729187] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 822.736837] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335746, 'name': CreateVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 822.864593] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bbe5e181-e612-47b5-9b55-8370f3f5fbb8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.874708] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27052c77-93ac-4fe6-9aa5-b1f80e938656 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.913031] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86fc1521-328e-4320-a54e-139d3b2f8a0f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.923497] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94fb36e5-f3a5-41f4-883d-d53e0b640bec {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.927675] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335744, 'name': Rename_Task, 'duration_secs': 1.169296} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 822.927980] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Powering on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 822.928690] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-79816fc7-8c09-455c-aa75-3321d3760cf5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.940425] env[62499]: DEBUG nova.compute.provider_tree [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 822.948241] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Waiting for the task: (returnval){ [ 822.948241] env[62499]: value = "task-1335747" [ 822.948241] env[62499]: _type = "Task" [ 822.948241] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 822.956572] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335747, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 822.982677] env[62499]: DEBUG oslo_concurrency.lockutils [req-e6409fd5-f369-4195-b07c-ab7909adf445 req-1001c65f-c7f1-4efe-9be0-8a89c5f44a54 service nova] Releasing lock "refresh_cache-261cbefa-fb2d-48da-a4fe-80b744a931f6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 822.983147] env[62499]: DEBUG oslo_concurrency.lockutils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Acquired lock "refresh_cache-261cbefa-fb2d-48da-a4fe-80b744a931f6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 822.983345] env[62499]: DEBUG nova.network.neutron [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 823.157837] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Releasing lock "refresh_cache-7bdd4371-b5d5-4053-93b1-75d1c5b9835b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 823.158193] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 823.158443] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 823.158622] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 823.172300] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335745, 'name': CopyVirtualDisk_Task} progress is 100%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 823.177937] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 823.239195] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335746, 'name': CreateVM_Task, 'duration_secs': 0.419131} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 823.239536] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Created VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 823.240243] env[62499]: DEBUG oslo_concurrency.lockutils [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 823.240574] env[62499]: DEBUG oslo_concurrency.lockutils [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquired lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 823.241311] env[62499]: DEBUG oslo_concurrency.lockutils [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 823.241760] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-24f4c146-b442-40ca-849a-b84261666fb9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 823.247516] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 823.247516] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]520ceb69-7dce-d4ff-3a7f-064e935d109d" [ 823.247516] env[62499]: _type = "Task" [ 823.247516] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 823.255871] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]520ceb69-7dce-d4ff-3a7f-064e935d109d, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 823.443918] env[62499]: DEBUG nova.scheduler.client.report [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 823.458427] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335747, 'name': PowerOnVM_Task} progress is 66%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 823.522927] env[62499]: DEBUG nova.network.neutron [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 823.622933] env[62499]: DEBUG nova.network.neutron [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 823.673241] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335745, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.509605} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 823.673511] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore2] 39bc8b3a-e9d9-48e3-a193-5fbdc2454346/39bc8b3a-e9d9-48e3-a193-5fbdc2454346.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 823.673722] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Extending root virtual disk to 1048576 {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 823.673967] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-df3e3e4a-caae-4a6a-a5a5-00044c77ce4f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 823.679349] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 823.681574] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 823.681574] env[62499]: value = "task-1335748" [ 823.681574] env[62499]: _type = "Task" [ 823.681574] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 823.691230] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335748, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 823.756387] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]520ceb69-7dce-d4ff-3a7f-064e935d109d, 'name': SearchDatastore_Task, 'duration_secs': 0.009881} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 823.756671] env[62499]: DEBUG oslo_concurrency.lockutils [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Releasing lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 823.756896] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Processing image 0422123c-16ce-4c9d-b1c6-0281fe0e54ad {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 823.757136] env[62499]: DEBUG oslo_concurrency.lockutils [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 823.757270] env[62499]: DEBUG oslo_concurrency.lockutils [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquired lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 823.757442] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 823.758015] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-ae2a1f8e-b258-44be-b7c3-27b994f6ce66 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 823.768516] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 823.768694] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=62499) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 823.769415] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-7c4e9bd1-e946-42e0-9d59-e652b78c8052 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 823.774882] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 823.774882] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]5208aecb-1652-615a-9826-73885d4fb036" [ 823.774882] env[62499]: _type = "Task" [ 823.774882] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 823.783128] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]5208aecb-1652-615a-9826-73885d4fb036, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 823.923190] env[62499]: DEBUG nova.compute.manager [req-736118eb-1ee9-469b-a1ff-432d4ce9315d req-f5756d99-8852-48c8-b993-aad7239331b4 service nova] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Received event network-vif-deleted-1863af9b-2409-4da1-a6dc-d2bb4324e5ec {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 823.952021] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.929s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 823.952021] env[62499]: ERROR nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 818ca52d-49e7-4e46-a1ac-50d975693883, please check neutron logs for more information. [ 823.952021] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Traceback (most recent call last): [ 823.952021] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 823.952021] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] self.driver.spawn(context, instance, image_meta, [ 823.952021] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 823.952021] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] self._vmops.spawn(context, instance, image_meta, injected_files, [ 823.952021] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 823.952021] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] vm_ref = self.build_virtual_machine(instance, [ 823.952418] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 823.952418] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] vif_infos = vmwarevif.get_vif_info(self._session, [ 823.952418] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 823.952418] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] for vif in network_info: [ 823.952418] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 823.952418] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] return self._sync_wrapper(fn, *args, **kwargs) [ 823.952418] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 823.952418] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] self.wait() [ 823.952418] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 823.952418] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] self[:] = self._gt.wait() [ 823.952418] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 823.952418] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] return self._exit_event.wait() [ 823.952418] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 823.952764] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] result = hub.switch() [ 823.952764] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 823.952764] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] return self.greenlet.switch() [ 823.952764] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 823.952764] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] result = function(*args, **kwargs) [ 823.952764] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 823.952764] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] return func(*args, **kwargs) [ 823.952764] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 823.952764] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] raise e [ 823.952764] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 823.952764] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] nwinfo = self.network_api.allocate_for_instance( [ 823.952764] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 823.952764] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] created_port_ids = self._update_ports_for_instance( [ 823.953136] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 823.953136] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] with excutils.save_and_reraise_exception(): [ 823.953136] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 823.953136] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] self.force_reraise() [ 823.953136] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 823.953136] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] raise self.value [ 823.953136] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 823.953136] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] updated_port = self._update_port( [ 823.953136] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 823.953136] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] _ensure_no_port_binding_failure(port) [ 823.953136] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 823.953136] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] raise exception.PortBindingFailed(port_id=port['id']) [ 823.953442] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] nova.exception.PortBindingFailed: Binding failed for port 818ca52d-49e7-4e46-a1ac-50d975693883, please check neutron logs for more information. [ 823.953442] env[62499]: ERROR nova.compute.manager [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] [ 823.953442] env[62499]: DEBUG nova.compute.utils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Binding failed for port 818ca52d-49e7-4e46-a1ac-50d975693883, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 823.953442] env[62499]: DEBUG oslo_concurrency.lockutils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 12.839s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 823.956237] env[62499]: INFO nova.compute.claims [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 823.958982] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Build of instance 63b76c39-11ca-4f1e-b336-2caceece1f72 was re-scheduled: Binding failed for port 818ca52d-49e7-4e46-a1ac-50d975693883, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 823.969043] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 823.969043] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquiring lock "refresh_cache-63b76c39-11ca-4f1e-b336-2caceece1f72" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 823.969043] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Acquired lock "refresh_cache-63b76c39-11ca-4f1e-b336-2caceece1f72" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 823.969043] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 823.976882] env[62499]: DEBUG oslo_vmware.api [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335747, 'name': PowerOnVM_Task, 'duration_secs': 0.580309} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 823.977144] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Powered on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 823.977975] env[62499]: INFO nova.compute.manager [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Took 6.39 seconds to spawn the instance on the hypervisor. [ 823.977975] env[62499]: DEBUG nova.compute.manager [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 823.979677] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-259d8d06-9c68-41a6-a384-0b4074488d2d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.126098] env[62499]: DEBUG oslo_concurrency.lockutils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Releasing lock "refresh_cache-261cbefa-fb2d-48da-a4fe-80b744a931f6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 824.126533] env[62499]: DEBUG nova.compute.manager [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 824.126721] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 824.127029] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-457c1f6b-2e14-4184-9743-10f1f250cdcb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.136934] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b73537c2-4b18-4a18-b08c-e8df740425ef {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.163188] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 261cbefa-fb2d-48da-a4fe-80b744a931f6 could not be found. [ 824.163423] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 824.163603] env[62499]: INFO nova.compute.manager [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Took 0.04 seconds to destroy the instance on the hypervisor. [ 824.163847] env[62499]: DEBUG oslo.service.loopingcall [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 824.164073] env[62499]: DEBUG nova.compute.manager [-] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 824.164250] env[62499]: DEBUG nova.network.neutron [-] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 824.181206] env[62499]: DEBUG nova.network.neutron [-] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 824.186019] env[62499]: INFO nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 7bdd4371-b5d5-4053-93b1-75d1c5b9835b] Took 1.02 seconds to deallocate network for instance. [ 824.196020] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335748, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.214257} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 824.196743] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Extended root virtual disk {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 824.197506] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f26cd0d-5042-477c-9624-f93e97bb7ebd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.217416] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Reconfiguring VM instance instance-00000038 to attach disk [datastore2] 39bc8b3a-e9d9-48e3-a193-5fbdc2454346/39bc8b3a-e9d9-48e3-a193-5fbdc2454346.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 824.217901] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-985f71fa-72ef-4248-98c9-7fce92894d2c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.239404] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 824.239404] env[62499]: value = "task-1335749" [ 824.239404] env[62499]: _type = "Task" [ 824.239404] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 824.247423] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335749, 'name': ReconfigVM_Task} progress is 5%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 824.286515] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]5208aecb-1652-615a-9826-73885d4fb036, 'name': SearchDatastore_Task, 'duration_secs': 0.051454} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 824.287354] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-dbdad709-50ff-4c95-825b-30d37250adeb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.293287] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 824.293287] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52d20bee-027f-ef28-3128-5a5fac1fac40" [ 824.293287] env[62499]: _type = "Task" [ 824.293287] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 824.301757] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52d20bee-027f-ef28-3128-5a5fac1fac40, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 824.498546] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 824.502614] env[62499]: INFO nova.compute.manager [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Took 27.79 seconds to build instance. [ 824.601887] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 824.686075] env[62499]: DEBUG nova.network.neutron [-] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 824.749961] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335749, 'name': ReconfigVM_Task, 'duration_secs': 0.396126} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 824.750140] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Reconfigured VM instance instance-00000038 to attach disk [datastore2] 39bc8b3a-e9d9-48e3-a193-5fbdc2454346/39bc8b3a-e9d9-48e3-a193-5fbdc2454346.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 824.750762] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-5739a7db-0c82-4156-afe4-a5c852ec563d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.759535] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 824.759535] env[62499]: value = "task-1335750" [ 824.759535] env[62499]: _type = "Task" [ 824.759535] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 824.766482] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335750, 'name': Rename_Task} progress is 5%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 824.804787] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52d20bee-027f-ef28-3128-5a5fac1fac40, 'name': SearchDatastore_Task, 'duration_secs': 0.067861} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 824.805299] env[62499]: DEBUG oslo_concurrency.lockutils [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Releasing lock "[datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 824.805462] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore1] 15f8e04f-8c14-4315-8686-da4db517f7b6/15f8e04f-8c14-4315-8686-da4db517f7b6.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 824.806189] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-6745f8d6-dac0-43ca-9f4d-b0d9c7bc43fb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.814064] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 824.814064] env[62499]: value = "task-1335751" [ 824.814064] env[62499]: _type = "Task" [ 824.814064] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 824.822708] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335751, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 825.004846] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3362da7f-2fe9-426a-98e0-39d630918f82 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Lock "57d21761-431b-4a41-86eb-038f8c35d8ef" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 150.629s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 825.109525] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Releasing lock "refresh_cache-63b76c39-11ca-4f1e-b336-2caceece1f72" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 825.109525] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 825.109525] env[62499]: DEBUG nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 825.109525] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 825.128629] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 825.192211] env[62499]: INFO nova.compute.manager [-] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Took 1.03 seconds to deallocate network for instance. [ 825.198258] env[62499]: DEBUG nova.compute.claims [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 825.198910] env[62499]: DEBUG oslo_concurrency.lockutils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 825.225200] env[62499]: INFO nova.scheduler.client.report [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Deleted allocations for instance 7bdd4371-b5d5-4053-93b1-75d1c5b9835b [ 825.277878] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335750, 'name': Rename_Task, 'duration_secs': 0.137694} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 825.277878] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Powering on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 825.277878] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-bb1bcb7f-e72d-4a55-bc37-bfc8f3347191 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.285977] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 825.285977] env[62499]: value = "task-1335752" [ 825.285977] env[62499]: _type = "Task" [ 825.285977] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 825.297575] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335752, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 825.325960] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335751, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.502371} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 825.326239] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore1] 15f8e04f-8c14-4315-8686-da4db517f7b6/15f8e04f-8c14-4315-8686-da4db517f7b6.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 825.326449] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Extending root virtual disk to 1048576 {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 825.326703] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-b790235b-1d9d-4511-a812-45faf6499eb2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.333933] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 825.333933] env[62499]: value = "task-1335753" [ 825.333933] env[62499]: _type = "Task" [ 825.333933] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 825.343961] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335753, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 825.403047] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4dab6a6e-68bf-4729-a40c-59829c331fff {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.410518] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f1209040-b909-41f3-8c3f-3dd58c8f97f0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.440013] env[62499]: DEBUG nova.compute.manager [None req-d4641bbb-9f08-4ef9-aceb-c5601207a799 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 825.441273] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d805fb18-34ea-4938-8a2a-2eb1b0d8a3ed {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.444324] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0d27140-6829-4bbf-a5d4-25b21e68ff6a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.456018] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67a8c7c3-6c62-4b8c-83c5-74be3ec0dfb2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.470247] env[62499]: DEBUG nova.compute.provider_tree [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 825.508170] env[62499]: DEBUG nova.compute.manager [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 825.622101] env[62499]: DEBUG oslo_concurrency.lockutils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Acquiring lock "a10b9328-394e-4673-928e-e6fa87b8c844" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 825.622342] env[62499]: DEBUG oslo_concurrency.lockutils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Lock "a10b9328-394e-4673-928e-e6fa87b8c844" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 825.634530] env[62499]: DEBUG nova.network.neutron [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 825.735717] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "7bdd4371-b5d5-4053-93b1-75d1c5b9835b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 165.987s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 825.796019] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335752, 'name': PowerOnVM_Task} progress is 66%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 825.844069] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335753, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.060927} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 825.844379] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Extended root virtual disk {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 825.845217] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c979993-8b56-43eb-a115-8424ce943182 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.857427] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Acquiring lock "57d21761-431b-4a41-86eb-038f8c35d8ef" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 825.857696] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Lock "57d21761-431b-4a41-86eb-038f8c35d8ef" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 825.857962] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Acquiring lock "57d21761-431b-4a41-86eb-038f8c35d8ef-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 825.858194] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Lock "57d21761-431b-4a41-86eb-038f8c35d8ef-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 825.858372] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Lock "57d21761-431b-4a41-86eb-038f8c35d8ef-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 825.869271] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Reconfiguring VM instance instance-00000039 to attach disk [datastore1] 15f8e04f-8c14-4315-8686-da4db517f7b6/15f8e04f-8c14-4315-8686-da4db517f7b6.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 825.869980] env[62499]: INFO nova.compute.manager [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Terminating instance [ 825.871580] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-a18b2957-6b06-45a4-94ce-c8ec38ba8936 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.887048] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Acquiring lock "refresh_cache-57d21761-431b-4a41-86eb-038f8c35d8ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 825.887213] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Acquired lock "refresh_cache-57d21761-431b-4a41-86eb-038f8c35d8ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 825.887382] env[62499]: DEBUG nova.network.neutron [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 825.895039] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 825.895039] env[62499]: value = "task-1335754" [ 825.895039] env[62499]: _type = "Task" [ 825.895039] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 825.903896] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335754, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 825.955200] env[62499]: INFO nova.compute.manager [None req-d4641bbb-9f08-4ef9-aceb-c5601207a799 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] instance snapshotting [ 825.955830] env[62499]: DEBUG nova.objects.instance [None req-d4641bbb-9f08-4ef9-aceb-c5601207a799 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Lazy-loading 'flavor' on Instance uuid 57d21761-431b-4a41-86eb-038f8c35d8ef {{(pid=62499) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 825.973024] env[62499]: DEBUG nova.scheduler.client.report [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 826.034590] env[62499]: DEBUG oslo_concurrency.lockutils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.136723] env[62499]: INFO nova.compute.manager [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] [instance: 63b76c39-11ca-4f1e-b336-2caceece1f72] Took 1.03 seconds to deallocate network for instance. [ 826.240286] env[62499]: DEBUG nova.compute.manager [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 826.296624] env[62499]: DEBUG oslo_vmware.api [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335752, 'name': PowerOnVM_Task, 'duration_secs': 0.860189} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 826.296898] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Powered on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 826.297132] env[62499]: DEBUG nova.compute.manager [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 826.298513] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44915a28-64f4-4122-89f4-8aa3bc60ced6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 826.405721] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335754, 'name': ReconfigVM_Task, 'duration_secs': 0.276754} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 826.405982] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Reconfigured VM instance instance-00000039 to attach disk [datastore1] 15f8e04f-8c14-4315-8686-da4db517f7b6/15f8e04f-8c14-4315-8686-da4db517f7b6.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 826.406600] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-0b870ab0-e2c0-4749-beff-7cac1fb3e457 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 826.409391] env[62499]: DEBUG nova.network.neutron [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 826.412902] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 826.412902] env[62499]: value = "task-1335755" [ 826.412902] env[62499]: _type = "Task" [ 826.412902] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 826.424261] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335755, 'name': Rename_Task} progress is 5%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 826.461845] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3cc86092-a4d3-4de7-a232-720f3348ab53 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 826.478810] env[62499]: DEBUG oslo_concurrency.lockutils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.527s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 826.479382] env[62499]: DEBUG nova.compute.manager [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 826.482243] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.055s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.483817] env[62499]: INFO nova.compute.claims [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 826.486647] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b4dbeb8-1abf-43b4-b8dc-0e29dcaba400 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 826.490542] env[62499]: DEBUG nova.network.neutron [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 826.759217] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.814504] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.923643] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335755, 'name': Rename_Task, 'duration_secs': 0.133798} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 826.923919] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Powering on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 826.924173] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-a91c5346-dd2c-4e5a-8234-a36e7bfb6be3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 826.930312] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 826.930312] env[62499]: value = "task-1335756" [ 826.930312] env[62499]: _type = "Task" [ 826.930312] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 826.938377] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335756, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 826.993740] env[62499]: DEBUG nova.compute.utils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 826.995630] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Releasing lock "refresh_cache-57d21761-431b-4a41-86eb-038f8c35d8ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 826.996018] env[62499]: DEBUG nova.compute.manager [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 826.996212] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 826.996504] env[62499]: DEBUG nova.compute.manager [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 826.996659] env[62499]: DEBUG nova.network.neutron [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 826.999339] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-924f2b6c-24fc-460c-aab7-0b79cfe4b50b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 827.002654] env[62499]: DEBUG nova.compute.manager [None req-d4641bbb-9f08-4ef9-aceb-c5601207a799 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Instance disappeared during snapshot {{(pid=62499) _snapshot_instance /opt/stack/nova/nova/compute/manager.py:4494}} [ 827.011204] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Powering off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 827.011448] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-cb238d55-349d-4421-b1f2-af61fe30c530 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 827.016983] env[62499]: DEBUG oslo_vmware.api [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Waiting for the task: (returnval){ [ 827.016983] env[62499]: value = "task-1335757" [ 827.016983] env[62499]: _type = "Task" [ 827.016983] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 827.029650] env[62499]: DEBUG oslo_vmware.api [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335757, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 827.069153] env[62499]: DEBUG nova.policy [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2fedbea4e5b945918a1e906af369211f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '4f1e8a17916f4b179a86748cb34ec51d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 827.138104] env[62499]: DEBUG nova.compute.manager [None req-d4641bbb-9f08-4ef9-aceb-c5601207a799 tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Found 0 images (rotation: 2) {{(pid=62499) _rotate_backups /opt/stack/nova/nova/compute/manager.py:4554}} [ 827.170335] env[62499]: INFO nova.scheduler.client.report [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Deleted allocations for instance 63b76c39-11ca-4f1e-b336-2caceece1f72 [ 827.352634] env[62499]: DEBUG nova.network.neutron [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Successfully created port: 80575751-bae4-48a6-b43b-1a4c37d4b3fd {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 827.359896] env[62499]: DEBUG oslo_concurrency.lockutils [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "39bc8b3a-e9d9-48e3-a193-5fbdc2454346" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 827.359896] env[62499]: DEBUG oslo_concurrency.lockutils [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "39bc8b3a-e9d9-48e3-a193-5fbdc2454346" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 827.359896] env[62499]: DEBUG oslo_concurrency.lockutils [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "39bc8b3a-e9d9-48e3-a193-5fbdc2454346-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 827.359896] env[62499]: DEBUG oslo_concurrency.lockutils [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "39bc8b3a-e9d9-48e3-a193-5fbdc2454346-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 827.360232] env[62499]: DEBUG oslo_concurrency.lockutils [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "39bc8b3a-e9d9-48e3-a193-5fbdc2454346-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 827.361085] env[62499]: INFO nova.compute.manager [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Terminating instance [ 827.362654] env[62499]: DEBUG oslo_concurrency.lockutils [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "refresh_cache-39bc8b3a-e9d9-48e3-a193-5fbdc2454346" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 827.362811] env[62499]: DEBUG oslo_concurrency.lockutils [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquired lock "refresh_cache-39bc8b3a-e9d9-48e3-a193-5fbdc2454346" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 827.362974] env[62499]: DEBUG nova.network.neutron [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 827.441505] env[62499]: DEBUG oslo_vmware.api [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335756, 'name': PowerOnVM_Task, 'duration_secs': 0.430876} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 827.441779] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Powered on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 827.441981] env[62499]: DEBUG nova.compute.manager [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 827.442757] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c4d8376-71a6-4a06-9796-2637356912fa {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 827.496906] env[62499]: DEBUG nova.compute.manager [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 827.529680] env[62499]: DEBUG oslo_vmware.api [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335757, 'name': PowerOffVM_Task, 'duration_secs': 0.189284} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 827.530425] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Powered off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 827.530425] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Unregistering the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 827.530425] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-42dce317-6f0c-4afa-b529-79e495bea626 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 827.551357] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Unregistered the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 827.551613] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Deleting contents of the VM from datastore datastore2 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 827.551736] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Deleting the datastore file [datastore2] 57d21761-431b-4a41-86eb-038f8c35d8ef {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 827.551978] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-82adb3e5-83c3-43c4-ad95-694e8d552f15 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 827.560759] env[62499]: DEBUG oslo_vmware.api [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Waiting for the task: (returnval){ [ 827.560759] env[62499]: value = "task-1335759" [ 827.560759] env[62499]: _type = "Task" [ 827.560759] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 827.570805] env[62499]: DEBUG oslo_vmware.api [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335759, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 827.679310] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ecd0d129-2452-4890-8e73-192545d63492 tempest-MultipleCreateTestJSON-988372643 tempest-MultipleCreateTestJSON-988372643-project-member] Lock "63b76c39-11ca-4f1e-b336-2caceece1f72" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 167.877s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 827.804017] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bcd161e0-2240-4374-8a23-7d91f8f50e48 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 827.810672] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12b7536f-63f0-48d8-b8db-05eebd9b1f73 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 827.845480] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bfea8b59-4b2d-4d1e-a9ca-da0f20b7a807 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 827.854533] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-43275810-1732-47de-b783-fa9f60694ce8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 827.875211] env[62499]: DEBUG nova.compute.provider_tree [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 827.892021] env[62499]: DEBUG nova.network.neutron [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 827.958497] env[62499]: DEBUG nova.network.neutron [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 827.964406] env[62499]: DEBUG oslo_concurrency.lockutils [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 828.070287] env[62499]: DEBUG oslo_vmware.api [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Task: {'id': task-1335759, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.315334} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 828.071426] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Deleted the datastore file {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 828.071426] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Deleted contents of the VM from datastore datastore2 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 828.071426] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 828.071426] env[62499]: INFO nova.compute.manager [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Took 1.07 seconds to destroy the instance on the hypervisor. [ 828.071426] env[62499]: DEBUG oslo.service.loopingcall [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 828.071646] env[62499]: DEBUG nova.compute.manager [-] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 828.071646] env[62499]: DEBUG nova.network.neutron [-] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 828.087459] env[62499]: DEBUG nova.network.neutron [-] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 828.139902] env[62499]: DEBUG nova.compute.manager [req-12aad97f-6c1f-4fc5-aa3b-74d78cc38d1a req-4f410382-1751-4b35-b8ed-731537bb28a1 service nova] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Received event network-changed-80575751-bae4-48a6-b43b-1a4c37d4b3fd {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 828.139902] env[62499]: DEBUG nova.compute.manager [req-12aad97f-6c1f-4fc5-aa3b-74d78cc38d1a req-4f410382-1751-4b35-b8ed-731537bb28a1 service nova] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Refreshing instance network info cache due to event network-changed-80575751-bae4-48a6-b43b-1a4c37d4b3fd. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 828.139902] env[62499]: DEBUG oslo_concurrency.lockutils [req-12aad97f-6c1f-4fc5-aa3b-74d78cc38d1a req-4f410382-1751-4b35-b8ed-731537bb28a1 service nova] Acquiring lock "refresh_cache-49f9108c-7256-4a2c-9ffd-a6d041a180e4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 828.140250] env[62499]: DEBUG oslo_concurrency.lockutils [req-12aad97f-6c1f-4fc5-aa3b-74d78cc38d1a req-4f410382-1751-4b35-b8ed-731537bb28a1 service nova] Acquired lock "refresh_cache-49f9108c-7256-4a2c-9ffd-a6d041a180e4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 828.140250] env[62499]: DEBUG nova.network.neutron [req-12aad97f-6c1f-4fc5-aa3b-74d78cc38d1a req-4f410382-1751-4b35-b8ed-731537bb28a1 service nova] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Refreshing network info cache for port 80575751-bae4-48a6-b43b-1a4c37d4b3fd {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 828.184640] env[62499]: DEBUG nova.compute.manager [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 828.310857] env[62499]: ERROR nova.compute.manager [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 80575751-bae4-48a6-b43b-1a4c37d4b3fd, please check neutron logs for more information. [ 828.310857] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 828.310857] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 828.310857] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 828.310857] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 828.310857] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 828.310857] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 828.310857] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 828.310857] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 828.310857] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 828.310857] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 828.310857] env[62499]: ERROR nova.compute.manager raise self.value [ 828.310857] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 828.310857] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 828.310857] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 828.310857] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 828.311351] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 828.311351] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 828.311351] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 80575751-bae4-48a6-b43b-1a4c37d4b3fd, please check neutron logs for more information. [ 828.311351] env[62499]: ERROR nova.compute.manager [ 828.311351] env[62499]: Traceback (most recent call last): [ 828.311351] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 828.311351] env[62499]: listener.cb(fileno) [ 828.311351] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 828.311351] env[62499]: result = function(*args, **kwargs) [ 828.311351] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 828.311351] env[62499]: return func(*args, **kwargs) [ 828.311351] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 828.311351] env[62499]: raise e [ 828.311351] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 828.311351] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 828.311351] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 828.311351] env[62499]: created_port_ids = self._update_ports_for_instance( [ 828.311351] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 828.311351] env[62499]: with excutils.save_and_reraise_exception(): [ 828.311351] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 828.311351] env[62499]: self.force_reraise() [ 828.311351] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 828.311351] env[62499]: raise self.value [ 828.311351] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 828.311351] env[62499]: updated_port = self._update_port( [ 828.311351] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 828.311351] env[62499]: _ensure_no_port_binding_failure(port) [ 828.311351] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 828.311351] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 828.312030] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 80575751-bae4-48a6-b43b-1a4c37d4b3fd, please check neutron logs for more information. [ 828.312030] env[62499]: Removing descriptor: 16 [ 828.381366] env[62499]: DEBUG nova.scheduler.client.report [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 828.462434] env[62499]: DEBUG oslo_concurrency.lockutils [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Releasing lock "refresh_cache-39bc8b3a-e9d9-48e3-a193-5fbdc2454346" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 828.462935] env[62499]: DEBUG nova.compute.manager [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 828.463291] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 828.464361] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-575de219-4696-4a8c-960d-5dc77660f832 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.473069] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Powering off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 828.473581] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-71743cc0-2589-46f3-b21c-2f1aa82eb3bf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.480247] env[62499]: DEBUG oslo_vmware.api [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 828.480247] env[62499]: value = "task-1335760" [ 828.480247] env[62499]: _type = "Task" [ 828.480247] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 828.489324] env[62499]: DEBUG oslo_vmware.api [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335760, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 828.510516] env[62499]: DEBUG nova.compute.manager [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 828.539105] env[62499]: DEBUG nova.virt.hardware [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 828.539105] env[62499]: DEBUG nova.virt.hardware [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 828.539105] env[62499]: DEBUG nova.virt.hardware [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 828.539265] env[62499]: DEBUG nova.virt.hardware [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 828.539265] env[62499]: DEBUG nova.virt.hardware [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 828.539265] env[62499]: DEBUG nova.virt.hardware [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 828.539749] env[62499]: DEBUG nova.virt.hardware [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 828.541846] env[62499]: DEBUG nova.virt.hardware [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 828.542186] env[62499]: DEBUG nova.virt.hardware [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 828.542494] env[62499]: DEBUG nova.virt.hardware [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 828.545027] env[62499]: DEBUG nova.virt.hardware [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 828.545027] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1b721b7-ae9c-4827-82cf-0ee421144523 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.552402] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-781441b3-1898-49dc-968c-1de09b3640b3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.568582] env[62499]: ERROR nova.compute.manager [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 80575751-bae4-48a6-b43b-1a4c37d4b3fd, please check neutron logs for more information. [ 828.568582] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Traceback (most recent call last): [ 828.568582] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 828.568582] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] yield resources [ 828.568582] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 828.568582] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] self.driver.spawn(context, instance, image_meta, [ 828.568582] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 828.568582] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 828.568582] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 828.568582] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] vm_ref = self.build_virtual_machine(instance, [ 828.568582] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 828.568952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] vif_infos = vmwarevif.get_vif_info(self._session, [ 828.568952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 828.568952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] for vif in network_info: [ 828.568952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 828.568952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] return self._sync_wrapper(fn, *args, **kwargs) [ 828.568952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 828.568952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] self.wait() [ 828.568952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 828.568952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] self[:] = self._gt.wait() [ 828.568952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 828.568952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] return self._exit_event.wait() [ 828.568952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 828.568952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] current.throw(*self._exc) [ 828.569339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 828.569339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] result = function(*args, **kwargs) [ 828.569339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 828.569339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] return func(*args, **kwargs) [ 828.569339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 828.569339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] raise e [ 828.569339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 828.569339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] nwinfo = self.network_api.allocate_for_instance( [ 828.569339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 828.569339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] created_port_ids = self._update_ports_for_instance( [ 828.569339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 828.569339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] with excutils.save_and_reraise_exception(): [ 828.569339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 828.569669] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] self.force_reraise() [ 828.569669] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 828.569669] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] raise self.value [ 828.569669] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 828.569669] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] updated_port = self._update_port( [ 828.569669] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 828.569669] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] _ensure_no_port_binding_failure(port) [ 828.569669] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 828.569669] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] raise exception.PortBindingFailed(port_id=port['id']) [ 828.569669] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] nova.exception.PortBindingFailed: Binding failed for port 80575751-bae4-48a6-b43b-1a4c37d4b3fd, please check neutron logs for more information. [ 828.569669] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] [ 828.569669] env[62499]: INFO nova.compute.manager [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Terminating instance [ 828.573793] env[62499]: DEBUG oslo_concurrency.lockutils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Acquiring lock "refresh_cache-49f9108c-7256-4a2c-9ffd-a6d041a180e4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 828.592508] env[62499]: DEBUG nova.network.neutron [-] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 828.658242] env[62499]: DEBUG nova.network.neutron [req-12aad97f-6c1f-4fc5-aa3b-74d78cc38d1a req-4f410382-1751-4b35-b8ed-731537bb28a1 service nova] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 828.710718] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 828.731664] env[62499]: DEBUG nova.network.neutron [req-12aad97f-6c1f-4fc5-aa3b-74d78cc38d1a req-4f410382-1751-4b35-b8ed-731537bb28a1 service nova] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 828.885779] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.403s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 828.886163] env[62499]: DEBUG nova.compute.manager [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 828.890541] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 10.966s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 828.892293] env[62499]: INFO nova.compute.claims [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 828.938687] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquiring lock "15f8e04f-8c14-4315-8686-da4db517f7b6" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 828.939294] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Lock "15f8e04f-8c14-4315-8686-da4db517f7b6" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 828.939607] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquiring lock "15f8e04f-8c14-4315-8686-da4db517f7b6-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 828.939928] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Lock "15f8e04f-8c14-4315-8686-da4db517f7b6-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 828.940203] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Lock "15f8e04f-8c14-4315-8686-da4db517f7b6-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 828.942868] env[62499]: INFO nova.compute.manager [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Terminating instance [ 828.946052] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquiring lock "refresh_cache-15f8e04f-8c14-4315-8686-da4db517f7b6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 828.946198] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquired lock "refresh_cache-15f8e04f-8c14-4315-8686-da4db517f7b6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 828.946372] env[62499]: DEBUG nova.network.neutron [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 828.996634] env[62499]: DEBUG oslo_vmware.api [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335760, 'name': PowerOffVM_Task, 'duration_secs': 0.198251} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 828.996634] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Powered off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 828.996833] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Unregistering the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 828.997040] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-38375cf6-a9d2-4100-98ad-f6a3ecb64f65 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 829.018773] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Unregistered the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 829.018988] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Deleting contents of the VM from datastore datastore2 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 829.019189] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Deleting the datastore file [datastore2] 39bc8b3a-e9d9-48e3-a193-5fbdc2454346 {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 829.019442] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-8a13134a-ab6c-4191-80e0-16625da39b8a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 829.025577] env[62499]: DEBUG oslo_vmware.api [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 829.025577] env[62499]: value = "task-1335762" [ 829.025577] env[62499]: _type = "Task" [ 829.025577] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 829.033514] env[62499]: DEBUG oslo_vmware.api [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335762, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 829.096820] env[62499]: INFO nova.compute.manager [-] [instance: 57d21761-431b-4a41-86eb-038f8c35d8ef] Took 1.03 seconds to deallocate network for instance. [ 829.235042] env[62499]: DEBUG oslo_concurrency.lockutils [req-12aad97f-6c1f-4fc5-aa3b-74d78cc38d1a req-4f410382-1751-4b35-b8ed-731537bb28a1 service nova] Releasing lock "refresh_cache-49f9108c-7256-4a2c-9ffd-a6d041a180e4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 829.235436] env[62499]: DEBUG oslo_concurrency.lockutils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Acquired lock "refresh_cache-49f9108c-7256-4a2c-9ffd-a6d041a180e4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 829.235620] env[62499]: DEBUG nova.network.neutron [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 829.399151] env[62499]: DEBUG nova.compute.utils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 829.400618] env[62499]: DEBUG nova.compute.manager [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 829.402992] env[62499]: DEBUG nova.network.neutron [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 829.442367] env[62499]: DEBUG nova.policy [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ea369595aaf8424e88ef45f27900f1f2', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5bbbd84912164655851b6e29356849c5', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 829.471186] env[62499]: DEBUG nova.network.neutron [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 829.524495] env[62499]: DEBUG nova.network.neutron [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 829.536224] env[62499]: DEBUG oslo_vmware.api [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335762, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.082042} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 829.536528] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Deleted the datastore file {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 829.536691] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Deleted contents of the VM from datastore datastore2 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 829.536836] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 829.537103] env[62499]: INFO nova.compute.manager [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Took 1.07 seconds to destroy the instance on the hypervisor. [ 829.537359] env[62499]: DEBUG oslo.service.loopingcall [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 829.538183] env[62499]: DEBUG nova.compute.manager [-] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 829.538302] env[62499]: DEBUG nova.network.neutron [-] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 829.561153] env[62499]: DEBUG nova.network.neutron [-] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 829.603629] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 829.753695] env[62499]: DEBUG nova.network.neutron [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 829.786770] env[62499]: DEBUG nova.network.neutron [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Successfully created port: b80023b9-103c-4a15-948f-4139319d4feb {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 829.811710] env[62499]: DEBUG nova.network.neutron [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 829.905048] env[62499]: DEBUG nova.compute.manager [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 830.030810] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Releasing lock "refresh_cache-15f8e04f-8c14-4315-8686-da4db517f7b6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 830.031310] env[62499]: DEBUG nova.compute.manager [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 830.031521] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 830.032477] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92b76719-6cff-4030-aa04-36bb854444fb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.047210] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Powering off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 830.047210] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-d046bcfb-b0e8-4f45-aff5-0b6e85f0d9bc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.054130] env[62499]: DEBUG oslo_vmware.api [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 830.054130] env[62499]: value = "task-1335763" [ 830.054130] env[62499]: _type = "Task" [ 830.054130] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 830.069398] env[62499]: DEBUG nova.network.neutron [-] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 830.070328] env[62499]: DEBUG oslo_vmware.api [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335763, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 830.170821] env[62499]: DEBUG nova.compute.manager [req-69fb5ca1-d226-4f73-bda2-e3d98cff7c83 req-6542860b-ea46-4ad0-9609-3711bd5fad2a service nova] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Received event network-vif-deleted-80575751-bae4-48a6-b43b-1a4c37d4b3fd {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 830.240934] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d69fe93d-dd5b-49d2-be47-221553ed5b1e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.248587] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c63a7eb-fa46-49bb-a507-2c98bbe8a756 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.277821] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f212bf7c-4318-4729-b95c-cc69289fc532 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.285610] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b3e3295-4e07-41e1-bf43-1748ccc95f06 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.299450] env[62499]: DEBUG nova.compute.provider_tree [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 830.313982] env[62499]: DEBUG oslo_concurrency.lockutils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Releasing lock "refresh_cache-49f9108c-7256-4a2c-9ffd-a6d041a180e4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 830.314457] env[62499]: DEBUG nova.compute.manager [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 830.314657] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 830.314932] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2bea2943-a744-4193-a017-bab120f9a353 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.324184] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-266e16d7-8ed7-40ff-9de2-de9818e03f02 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.349975] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 49f9108c-7256-4a2c-9ffd-a6d041a180e4 could not be found. [ 830.350239] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 830.350430] env[62499]: INFO nova.compute.manager [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Took 0.04 seconds to destroy the instance on the hypervisor. [ 830.350674] env[62499]: DEBUG oslo.service.loopingcall [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 830.350921] env[62499]: DEBUG nova.compute.manager [-] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 830.351042] env[62499]: DEBUG nova.network.neutron [-] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 830.381976] env[62499]: DEBUG nova.network.neutron [-] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 830.569012] env[62499]: DEBUG oslo_vmware.api [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335763, 'name': PowerOffVM_Task, 'duration_secs': 0.185239} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 830.569426] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Powered off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 830.569636] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Unregistering the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 830.570757] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-d0278767-a953-4e03-ae74-d5e38cc0602a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.572504] env[62499]: INFO nova.compute.manager [-] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Took 1.03 seconds to deallocate network for instance. [ 830.600880] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Unregistered the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 830.600880] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Deleting contents of the VM from datastore datastore1 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 830.600880] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Deleting the datastore file [datastore1] 15f8e04f-8c14-4315-8686-da4db517f7b6 {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 830.601510] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-0b1f07c6-596c-4c7d-8dec-9cf693f67aff {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.608765] env[62499]: DEBUG oslo_vmware.api [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for the task: (returnval){ [ 830.608765] env[62499]: value = "task-1335765" [ 830.608765] env[62499]: _type = "Task" [ 830.608765] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 830.617696] env[62499]: DEBUG oslo_vmware.api [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335765, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 830.802662] env[62499]: DEBUG nova.scheduler.client.report [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 830.813075] env[62499]: ERROR nova.compute.manager [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b80023b9-103c-4a15-948f-4139319d4feb, please check neutron logs for more information. [ 830.813075] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 830.813075] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 830.813075] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 830.813075] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 830.813075] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 830.813075] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 830.813075] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 830.813075] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 830.813075] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 830.813075] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 830.813075] env[62499]: ERROR nova.compute.manager raise self.value [ 830.813075] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 830.813075] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 830.813075] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 830.813075] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 830.814384] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 830.814384] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 830.814384] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b80023b9-103c-4a15-948f-4139319d4feb, please check neutron logs for more information. [ 830.814384] env[62499]: ERROR nova.compute.manager [ 830.814384] env[62499]: Traceback (most recent call last): [ 830.814384] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 830.814384] env[62499]: listener.cb(fileno) [ 830.814384] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 830.814384] env[62499]: result = function(*args, **kwargs) [ 830.814384] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 830.814384] env[62499]: return func(*args, **kwargs) [ 830.814384] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 830.814384] env[62499]: raise e [ 830.814384] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 830.814384] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 830.814384] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 830.814384] env[62499]: created_port_ids = self._update_ports_for_instance( [ 830.814384] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 830.814384] env[62499]: with excutils.save_and_reraise_exception(): [ 830.814384] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 830.814384] env[62499]: self.force_reraise() [ 830.814384] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 830.814384] env[62499]: raise self.value [ 830.814384] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 830.814384] env[62499]: updated_port = self._update_port( [ 830.814384] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 830.814384] env[62499]: _ensure_no_port_binding_failure(port) [ 830.814384] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 830.814384] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 830.815134] env[62499]: nova.exception.PortBindingFailed: Binding failed for port b80023b9-103c-4a15-948f-4139319d4feb, please check neutron logs for more information. [ 830.815134] env[62499]: Removing descriptor: 16 [ 830.885167] env[62499]: DEBUG nova.network.neutron [-] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 830.917428] env[62499]: DEBUG nova.compute.manager [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 830.943297] env[62499]: DEBUG nova.virt.hardware [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 830.944895] env[62499]: DEBUG nova.virt.hardware [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 830.945165] env[62499]: DEBUG nova.virt.hardware [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 830.945380] env[62499]: DEBUG nova.virt.hardware [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 830.945532] env[62499]: DEBUG nova.virt.hardware [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 830.945702] env[62499]: DEBUG nova.virt.hardware [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 830.945942] env[62499]: DEBUG nova.virt.hardware [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 830.946163] env[62499]: DEBUG nova.virt.hardware [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 830.946358] env[62499]: DEBUG nova.virt.hardware [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 830.946549] env[62499]: DEBUG nova.virt.hardware [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 830.946731] env[62499]: DEBUG nova.virt.hardware [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 830.947650] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c19f97b-50d7-43c9-af51-1a08adad527d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.956879] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-290c14fb-6ff3-4c70-a45f-376a4074ba20 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.970940] env[62499]: ERROR nova.compute.manager [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b80023b9-103c-4a15-948f-4139319d4feb, please check neutron logs for more information. [ 830.970940] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Traceback (most recent call last): [ 830.970940] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 830.970940] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] yield resources [ 830.970940] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 830.970940] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] self.driver.spawn(context, instance, image_meta, [ 830.970940] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 830.970940] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 830.970940] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 830.970940] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] vm_ref = self.build_virtual_machine(instance, [ 830.970940] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 830.971308] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] vif_infos = vmwarevif.get_vif_info(self._session, [ 830.971308] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 830.971308] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] for vif in network_info: [ 830.971308] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 830.971308] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] return self._sync_wrapper(fn, *args, **kwargs) [ 830.971308] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 830.971308] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] self.wait() [ 830.971308] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 830.971308] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] self[:] = self._gt.wait() [ 830.971308] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 830.971308] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] return self._exit_event.wait() [ 830.971308] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 830.971308] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] current.throw(*self._exc) [ 830.971604] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 830.971604] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] result = function(*args, **kwargs) [ 830.971604] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 830.971604] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] return func(*args, **kwargs) [ 830.971604] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 830.971604] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] raise e [ 830.971604] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 830.971604] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] nwinfo = self.network_api.allocate_for_instance( [ 830.971604] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 830.971604] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] created_port_ids = self._update_ports_for_instance( [ 830.971604] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 830.971604] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] with excutils.save_and_reraise_exception(): [ 830.971604] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 830.971898] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] self.force_reraise() [ 830.971898] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 830.971898] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] raise self.value [ 830.971898] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 830.971898] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] updated_port = self._update_port( [ 830.971898] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 830.971898] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] _ensure_no_port_binding_failure(port) [ 830.971898] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 830.971898] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] raise exception.PortBindingFailed(port_id=port['id']) [ 830.971898] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] nova.exception.PortBindingFailed: Binding failed for port b80023b9-103c-4a15-948f-4139319d4feb, please check neutron logs for more information. [ 830.971898] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] [ 830.971898] env[62499]: INFO nova.compute.manager [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Terminating instance [ 830.973639] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Acquiring lock "refresh_cache-d650a1a5-3706-4682-a813-f85ea23098e2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 830.973788] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Acquired lock "refresh_cache-d650a1a5-3706-4682-a813-f85ea23098e2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 830.973981] env[62499]: DEBUG nova.network.neutron [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 831.080461] env[62499]: DEBUG oslo_concurrency.lockutils [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.120063] env[62499]: DEBUG oslo_vmware.api [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Task: {'id': task-1335765, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.098908} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 831.120327] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Deleted the datastore file {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 831.120514] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Deleted contents of the VM from datastore datastore1 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 831.120695] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 831.120925] env[62499]: INFO nova.compute.manager [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Took 1.09 seconds to destroy the instance on the hypervisor. [ 831.121214] env[62499]: DEBUG oslo.service.loopingcall [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 831.121413] env[62499]: DEBUG nova.compute.manager [-] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 831.121542] env[62499]: DEBUG nova.network.neutron [-] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 831.136486] env[62499]: DEBUG nova.network.neutron [-] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 831.309492] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.419s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 831.310355] env[62499]: DEBUG nova.compute.manager [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 831.313771] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 12.121s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 831.315551] env[62499]: INFO nova.compute.claims [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 831.391273] env[62499]: INFO nova.compute.manager [-] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Took 1.04 seconds to deallocate network for instance. [ 831.392582] env[62499]: DEBUG nova.compute.claims [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 831.392807] env[62499]: DEBUG oslo_concurrency.lockutils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.492179] env[62499]: DEBUG nova.network.neutron [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 831.568687] env[62499]: DEBUG nova.network.neutron [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 831.639275] env[62499]: DEBUG nova.network.neutron [-] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 831.820808] env[62499]: DEBUG nova.compute.utils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 831.824687] env[62499]: DEBUG nova.compute.manager [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 831.824861] env[62499]: DEBUG nova.network.neutron [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 831.887677] env[62499]: DEBUG nova.policy [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '65a9f926ab8643c88b9251aa1ceb949e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd1229f15ad8648da9cf8a3537795d9fa', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 832.071611] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Releasing lock "refresh_cache-d650a1a5-3706-4682-a813-f85ea23098e2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 832.072207] env[62499]: DEBUG nova.compute.manager [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 832.072369] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 832.072822] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3ecab3fc-1e03-4f2d-98ed-703cc1f828dd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.082436] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72e80d79-121d-4519-8ff0-b4eb80ba7b41 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.104383] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d650a1a5-3706-4682-a813-f85ea23098e2 could not be found. [ 832.104525] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 832.104706] env[62499]: INFO nova.compute.manager [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Took 0.03 seconds to destroy the instance on the hypervisor. [ 832.104945] env[62499]: DEBUG oslo.service.loopingcall [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 832.105295] env[62499]: DEBUG nova.compute.manager [-] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 832.105295] env[62499]: DEBUG nova.network.neutron [-] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 832.123324] env[62499]: DEBUG nova.network.neutron [-] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 832.147330] env[62499]: INFO nova.compute.manager [-] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Took 1.03 seconds to deallocate network for instance. [ 832.304336] env[62499]: DEBUG nova.compute.manager [req-b0393c78-5c99-4e3e-b267-992e3266d556 req-aaa329df-3698-4555-9ab0-8a4de49a2012 service nova] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Received event network-changed-b80023b9-103c-4a15-948f-4139319d4feb {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 832.304336] env[62499]: DEBUG nova.compute.manager [req-b0393c78-5c99-4e3e-b267-992e3266d556 req-aaa329df-3698-4555-9ab0-8a4de49a2012 service nova] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Refreshing instance network info cache due to event network-changed-b80023b9-103c-4a15-948f-4139319d4feb. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 832.304336] env[62499]: DEBUG oslo_concurrency.lockutils [req-b0393c78-5c99-4e3e-b267-992e3266d556 req-aaa329df-3698-4555-9ab0-8a4de49a2012 service nova] Acquiring lock "refresh_cache-d650a1a5-3706-4682-a813-f85ea23098e2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 832.304336] env[62499]: DEBUG oslo_concurrency.lockutils [req-b0393c78-5c99-4e3e-b267-992e3266d556 req-aaa329df-3698-4555-9ab0-8a4de49a2012 service nova] Acquired lock "refresh_cache-d650a1a5-3706-4682-a813-f85ea23098e2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 832.304336] env[62499]: DEBUG nova.network.neutron [req-b0393c78-5c99-4e3e-b267-992e3266d556 req-aaa329df-3698-4555-9ab0-8a4de49a2012 service nova] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Refreshing network info cache for port b80023b9-103c-4a15-948f-4139319d4feb {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 832.325319] env[62499]: DEBUG nova.compute.manager [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 832.349571] env[62499]: DEBUG nova.network.neutron [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Successfully created port: d8305010-f9eb-4feb-8f48-828f581c7b97 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 832.625262] env[62499]: DEBUG nova.network.neutron [-] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 832.653922] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 832.674741] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-560403ae-c8b0-4747-99c9-1efe62dae2e2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.683724] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9878c8fb-422e-4721-8d55-0a1b1954c68d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.725514] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ee34f03-e82a-44a1-9a19-0c24612b3ab9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.733704] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ce12a00-50b9-4bd7-9bc0-a7639ef2416b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.747204] env[62499]: DEBUG nova.compute.provider_tree [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 832.828180] env[62499]: DEBUG nova.network.neutron [req-b0393c78-5c99-4e3e-b267-992e3266d556 req-aaa329df-3698-4555-9ab0-8a4de49a2012 service nova] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 832.914901] env[62499]: DEBUG nova.network.neutron [req-b0393c78-5c99-4e3e-b267-992e3266d556 req-aaa329df-3698-4555-9ab0-8a4de49a2012 service nova] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 833.135675] env[62499]: INFO nova.compute.manager [-] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Took 1.03 seconds to deallocate network for instance. [ 833.135675] env[62499]: DEBUG nova.compute.claims [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 833.135675] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.250435] env[62499]: DEBUG nova.scheduler.client.report [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 833.337412] env[62499]: DEBUG nova.compute.manager [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 833.370068] env[62499]: DEBUG nova.virt.hardware [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 833.370350] env[62499]: DEBUG nova.virt.hardware [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 833.370579] env[62499]: DEBUG nova.virt.hardware [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 833.370817] env[62499]: DEBUG nova.virt.hardware [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 833.370977] env[62499]: DEBUG nova.virt.hardware [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 833.374363] env[62499]: DEBUG nova.virt.hardware [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 833.374610] env[62499]: DEBUG nova.virt.hardware [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 833.374783] env[62499]: DEBUG nova.virt.hardware [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 833.374967] env[62499]: DEBUG nova.virt.hardware [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 833.375161] env[62499]: DEBUG nova.virt.hardware [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 833.375337] env[62499]: DEBUG nova.virt.hardware [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 833.376281] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab425116-ebf0-4584-b848-3edc68151796 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 833.384560] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-832a5955-6c85-4826-8768-b2c36c95355d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 833.418035] env[62499]: DEBUG oslo_concurrency.lockutils [req-b0393c78-5c99-4e3e-b267-992e3266d556 req-aaa329df-3698-4555-9ab0-8a4de49a2012 service nova] Releasing lock "refresh_cache-d650a1a5-3706-4682-a813-f85ea23098e2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 833.418459] env[62499]: DEBUG nova.compute.manager [req-b0393c78-5c99-4e3e-b267-992e3266d556 req-aaa329df-3698-4555-9ab0-8a4de49a2012 service nova] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Received event network-vif-deleted-b80023b9-103c-4a15-948f-4139319d4feb {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 833.424026] env[62499]: ERROR nova.compute.manager [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d8305010-f9eb-4feb-8f48-828f581c7b97, please check neutron logs for more information. [ 833.424026] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 833.424026] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 833.424026] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 833.424026] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 833.424026] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 833.424026] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 833.424026] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 833.424026] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 833.424026] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 833.424026] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 833.424026] env[62499]: ERROR nova.compute.manager raise self.value [ 833.424026] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 833.424026] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 833.424026] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 833.424026] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 833.424641] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 833.424641] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 833.424641] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d8305010-f9eb-4feb-8f48-828f581c7b97, please check neutron logs for more information. [ 833.424641] env[62499]: ERROR nova.compute.manager [ 833.424641] env[62499]: Traceback (most recent call last): [ 833.424641] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 833.424641] env[62499]: listener.cb(fileno) [ 833.424641] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 833.424641] env[62499]: result = function(*args, **kwargs) [ 833.424641] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 833.424641] env[62499]: return func(*args, **kwargs) [ 833.424641] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 833.424641] env[62499]: raise e [ 833.424641] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 833.424641] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 833.424641] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 833.424641] env[62499]: created_port_ids = self._update_ports_for_instance( [ 833.424641] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 833.424641] env[62499]: with excutils.save_and_reraise_exception(): [ 833.424641] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 833.424641] env[62499]: self.force_reraise() [ 833.424641] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 833.424641] env[62499]: raise self.value [ 833.424641] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 833.424641] env[62499]: updated_port = self._update_port( [ 833.424641] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 833.424641] env[62499]: _ensure_no_port_binding_failure(port) [ 833.424641] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 833.424641] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 833.425351] env[62499]: nova.exception.PortBindingFailed: Binding failed for port d8305010-f9eb-4feb-8f48-828f581c7b97, please check neutron logs for more information. [ 833.425351] env[62499]: Removing descriptor: 16 [ 833.425351] env[62499]: ERROR nova.compute.manager [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d8305010-f9eb-4feb-8f48-828f581c7b97, please check neutron logs for more information. [ 833.425351] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Traceback (most recent call last): [ 833.425351] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 833.425351] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] yield resources [ 833.425351] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 833.425351] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] self.driver.spawn(context, instance, image_meta, [ 833.425351] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 833.425351] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] self._vmops.spawn(context, instance, image_meta, injected_files, [ 833.425351] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 833.425351] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] vm_ref = self.build_virtual_machine(instance, [ 833.425653] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 833.425653] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] vif_infos = vmwarevif.get_vif_info(self._session, [ 833.425653] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 833.425653] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] for vif in network_info: [ 833.425653] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 833.425653] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] return self._sync_wrapper(fn, *args, **kwargs) [ 833.425653] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 833.425653] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] self.wait() [ 833.425653] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 833.425653] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] self[:] = self._gt.wait() [ 833.425653] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 833.425653] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] return self._exit_event.wait() [ 833.425653] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 833.425983] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] result = hub.switch() [ 833.425983] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 833.425983] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] return self.greenlet.switch() [ 833.425983] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 833.425983] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] result = function(*args, **kwargs) [ 833.425983] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 833.425983] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] return func(*args, **kwargs) [ 833.425983] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 833.425983] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] raise e [ 833.425983] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 833.425983] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] nwinfo = self.network_api.allocate_for_instance( [ 833.425983] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 833.425983] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] created_port_ids = self._update_ports_for_instance( [ 833.426293] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 833.426293] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] with excutils.save_and_reraise_exception(): [ 833.426293] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 833.426293] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] self.force_reraise() [ 833.426293] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 833.426293] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] raise self.value [ 833.426293] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 833.426293] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] updated_port = self._update_port( [ 833.426293] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 833.426293] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] _ensure_no_port_binding_failure(port) [ 833.426293] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 833.426293] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] raise exception.PortBindingFailed(port_id=port['id']) [ 833.426640] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] nova.exception.PortBindingFailed: Binding failed for port d8305010-f9eb-4feb-8f48-828f581c7b97, please check neutron logs for more information. [ 833.426640] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] [ 833.426640] env[62499]: INFO nova.compute.manager [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Terminating instance [ 833.428052] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Acquiring lock "refresh_cache-60a4b8c3-9dc4-4867-adca-4503e61ce237" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 833.428214] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Acquired lock "refresh_cache-60a4b8c3-9dc4-4867-adca-4503e61ce237" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 833.428378] env[62499]: DEBUG nova.network.neutron [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 833.525116] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Acquiring lock "d308417b-47d0-423a-a603-56d9251ef818" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.525358] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Lock "d308417b-47d0-423a-a603-56d9251ef818" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.760751] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.447s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.761475] env[62499]: DEBUG nova.compute.manager [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 833.767905] env[62499]: DEBUG oslo_concurrency.lockutils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.601s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.770229] env[62499]: INFO nova.compute.claims [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 833.950104] env[62499]: DEBUG nova.network.neutron [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 834.052883] env[62499]: DEBUG nova.network.neutron [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 834.278544] env[62499]: DEBUG nova.compute.utils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 834.282891] env[62499]: DEBUG nova.compute.manager [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 834.282891] env[62499]: DEBUG nova.network.neutron [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 834.327438] env[62499]: DEBUG nova.compute.manager [req-743e5219-f215-4d7e-b8bc-92ca48ca9c10 req-b8e4ec91-80f0-4035-b51f-b3289c49ff81 service nova] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Received event network-changed-d8305010-f9eb-4feb-8f48-828f581c7b97 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 834.327637] env[62499]: DEBUG nova.compute.manager [req-743e5219-f215-4d7e-b8bc-92ca48ca9c10 req-b8e4ec91-80f0-4035-b51f-b3289c49ff81 service nova] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Refreshing instance network info cache due to event network-changed-d8305010-f9eb-4feb-8f48-828f581c7b97. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 834.327844] env[62499]: DEBUG oslo_concurrency.lockutils [req-743e5219-f215-4d7e-b8bc-92ca48ca9c10 req-b8e4ec91-80f0-4035-b51f-b3289c49ff81 service nova] Acquiring lock "refresh_cache-60a4b8c3-9dc4-4867-adca-4503e61ce237" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 834.329355] env[62499]: DEBUG nova.policy [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '65a9f926ab8643c88b9251aa1ceb949e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd1229f15ad8648da9cf8a3537795d9fa', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 834.556477] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Releasing lock "refresh_cache-60a4b8c3-9dc4-4867-adca-4503e61ce237" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 834.556477] env[62499]: DEBUG nova.compute.manager [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 834.556477] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 834.557193] env[62499]: DEBUG oslo_concurrency.lockutils [req-743e5219-f215-4d7e-b8bc-92ca48ca9c10 req-b8e4ec91-80f0-4035-b51f-b3289c49ff81 service nova] Acquired lock "refresh_cache-60a4b8c3-9dc4-4867-adca-4503e61ce237" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 834.557193] env[62499]: DEBUG nova.network.neutron [req-743e5219-f215-4d7e-b8bc-92ca48ca9c10 req-b8e4ec91-80f0-4035-b51f-b3289c49ff81 service nova] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Refreshing network info cache for port d8305010-f9eb-4feb-8f48-828f581c7b97 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 834.558056] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-93b06511-70af-4617-a526-06820a05f566 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 834.567422] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa117167-0ca4-4c27-80b8-458f4ddbf501 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 834.589265] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 60a4b8c3-9dc4-4867-adca-4503e61ce237 could not be found. [ 834.589483] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 834.589663] env[62499]: INFO nova.compute.manager [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Took 0.03 seconds to destroy the instance on the hypervisor. [ 834.589925] env[62499]: DEBUG oslo.service.loopingcall [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 834.590415] env[62499]: DEBUG nova.compute.manager [-] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 834.590518] env[62499]: DEBUG nova.network.neutron [-] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 834.593253] env[62499]: DEBUG nova.network.neutron [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Successfully created port: 69123df3-56d1-490c-9303-b6427e432f7f {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 834.605607] env[62499]: DEBUG nova.network.neutron [-] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 834.782765] env[62499]: DEBUG nova.compute.manager [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 835.098495] env[62499]: DEBUG nova.network.neutron [req-743e5219-f215-4d7e-b8bc-92ca48ca9c10 req-b8e4ec91-80f0-4035-b51f-b3289c49ff81 service nova] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 835.099666] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3dc0a69-f2d5-4ec9-a8fa-5c619b7c8157 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.109633] env[62499]: DEBUG nova.network.neutron [-] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 835.111422] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff4e4c7d-8dff-4c48-a4ec-d8166fa496ff {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.148062] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6428d691-7760-49fc-b847-97055265a60d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.157366] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a1cb4fbc-be21-4431-beec-776e7d1ccdea {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.178326] env[62499]: DEBUG nova.compute.provider_tree [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 835.225792] env[62499]: DEBUG nova.network.neutron [req-743e5219-f215-4d7e-b8bc-92ca48ca9c10 req-b8e4ec91-80f0-4035-b51f-b3289c49ff81 service nova] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 835.555765] env[62499]: ERROR nova.compute.manager [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 69123df3-56d1-490c-9303-b6427e432f7f, please check neutron logs for more information. [ 835.555765] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 835.555765] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 835.555765] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 835.555765] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 835.555765] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 835.555765] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 835.555765] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 835.555765] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 835.555765] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 835.555765] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 835.555765] env[62499]: ERROR nova.compute.manager raise self.value [ 835.555765] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 835.555765] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 835.555765] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 835.555765] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 835.556424] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 835.556424] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 835.556424] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 69123df3-56d1-490c-9303-b6427e432f7f, please check neutron logs for more information. [ 835.556424] env[62499]: ERROR nova.compute.manager [ 835.556424] env[62499]: Traceback (most recent call last): [ 835.556424] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 835.556424] env[62499]: listener.cb(fileno) [ 835.556424] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 835.556424] env[62499]: result = function(*args, **kwargs) [ 835.556424] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 835.556424] env[62499]: return func(*args, **kwargs) [ 835.556424] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 835.556424] env[62499]: raise e [ 835.556424] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 835.556424] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 835.556424] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 835.556424] env[62499]: created_port_ids = self._update_ports_for_instance( [ 835.556424] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 835.556424] env[62499]: with excutils.save_and_reraise_exception(): [ 835.556424] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 835.556424] env[62499]: self.force_reraise() [ 835.556424] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 835.556424] env[62499]: raise self.value [ 835.556424] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 835.556424] env[62499]: updated_port = self._update_port( [ 835.556424] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 835.556424] env[62499]: _ensure_no_port_binding_failure(port) [ 835.556424] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 835.556424] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 835.557229] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 69123df3-56d1-490c-9303-b6427e432f7f, please check neutron logs for more information. [ 835.557229] env[62499]: Removing descriptor: 16 [ 835.618200] env[62499]: INFO nova.compute.manager [-] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Took 1.03 seconds to deallocate network for instance. [ 835.620664] env[62499]: DEBUG nova.compute.claims [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 835.620972] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 835.679196] env[62499]: DEBUG nova.scheduler.client.report [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 835.729402] env[62499]: DEBUG oslo_concurrency.lockutils [req-743e5219-f215-4d7e-b8bc-92ca48ca9c10 req-b8e4ec91-80f0-4035-b51f-b3289c49ff81 service nova] Releasing lock "refresh_cache-60a4b8c3-9dc4-4867-adca-4503e61ce237" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 835.729402] env[62499]: DEBUG nova.compute.manager [req-743e5219-f215-4d7e-b8bc-92ca48ca9c10 req-b8e4ec91-80f0-4035-b51f-b3289c49ff81 service nova] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Received event network-vif-deleted-d8305010-f9eb-4feb-8f48-828f581c7b97 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 835.791662] env[62499]: DEBUG nova.compute.manager [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 835.873485] env[62499]: DEBUG nova.virt.hardware [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 835.873728] env[62499]: DEBUG nova.virt.hardware [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 835.873886] env[62499]: DEBUG nova.virt.hardware [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 835.874089] env[62499]: DEBUG nova.virt.hardware [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 835.874243] env[62499]: DEBUG nova.virt.hardware [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 835.874395] env[62499]: DEBUG nova.virt.hardware [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 835.874640] env[62499]: DEBUG nova.virt.hardware [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 835.874828] env[62499]: DEBUG nova.virt.hardware [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 835.875020] env[62499]: DEBUG nova.virt.hardware [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 835.875194] env[62499]: DEBUG nova.virt.hardware [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 835.875416] env[62499]: DEBUG nova.virt.hardware [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 835.876345] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7e73a55-fd57-4140-9db7-d939e5162fbb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.884295] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1e265a0f-eb1c-402f-9cd1-38048d748a66 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.898030] env[62499]: ERROR nova.compute.manager [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 69123df3-56d1-490c-9303-b6427e432f7f, please check neutron logs for more information. [ 835.898030] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] Traceback (most recent call last): [ 835.898030] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 835.898030] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] yield resources [ 835.898030] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 835.898030] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] self.driver.spawn(context, instance, image_meta, [ 835.898030] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 835.898030] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] self._vmops.spawn(context, instance, image_meta, injected_files, [ 835.898030] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 835.898030] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] vm_ref = self.build_virtual_machine(instance, [ 835.898030] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 835.898392] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] vif_infos = vmwarevif.get_vif_info(self._session, [ 835.898392] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 835.898392] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] for vif in network_info: [ 835.898392] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 835.898392] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] return self._sync_wrapper(fn, *args, **kwargs) [ 835.898392] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 835.898392] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] self.wait() [ 835.898392] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 835.898392] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] self[:] = self._gt.wait() [ 835.898392] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 835.898392] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] return self._exit_event.wait() [ 835.898392] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 835.898392] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] current.throw(*self._exc) [ 835.899190] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 835.899190] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] result = function(*args, **kwargs) [ 835.899190] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 835.899190] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] return func(*args, **kwargs) [ 835.899190] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 835.899190] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] raise e [ 835.899190] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 835.899190] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] nwinfo = self.network_api.allocate_for_instance( [ 835.899190] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 835.899190] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] created_port_ids = self._update_ports_for_instance( [ 835.899190] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 835.899190] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] with excutils.save_and_reraise_exception(): [ 835.899190] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 835.899727] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] self.force_reraise() [ 835.899727] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 835.899727] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] raise self.value [ 835.899727] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 835.899727] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] updated_port = self._update_port( [ 835.899727] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 835.899727] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] _ensure_no_port_binding_failure(port) [ 835.899727] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 835.899727] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] raise exception.PortBindingFailed(port_id=port['id']) [ 835.899727] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] nova.exception.PortBindingFailed: Binding failed for port 69123df3-56d1-490c-9303-b6427e432f7f, please check neutron logs for more information. [ 835.899727] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] [ 835.899727] env[62499]: INFO nova.compute.manager [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Terminating instance [ 835.900625] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Acquiring lock "refresh_cache-ad749135-08c1-48ca-af56-300d6e796012" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 835.900885] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Acquired lock "refresh_cache-ad749135-08c1-48ca-af56-300d6e796012" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 835.900885] env[62499]: DEBUG nova.network.neutron [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 836.184836] env[62499]: DEBUG oslo_concurrency.lockutils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.417s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 836.185665] env[62499]: DEBUG nova.compute.manager [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 836.188664] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.008s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 836.190122] env[62499]: INFO nova.compute.claims [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 836.351195] env[62499]: DEBUG nova.compute.manager [req-79efc774-46c7-4351-ac4b-4a85ad8031fa req-10faed45-d78e-4452-9e70-a51d27ce9373 service nova] [instance: ad749135-08c1-48ca-af56-300d6e796012] Received event network-changed-69123df3-56d1-490c-9303-b6427e432f7f {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 836.351405] env[62499]: DEBUG nova.compute.manager [req-79efc774-46c7-4351-ac4b-4a85ad8031fa req-10faed45-d78e-4452-9e70-a51d27ce9373 service nova] [instance: ad749135-08c1-48ca-af56-300d6e796012] Refreshing instance network info cache due to event network-changed-69123df3-56d1-490c-9303-b6427e432f7f. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 836.351599] env[62499]: DEBUG oslo_concurrency.lockutils [req-79efc774-46c7-4351-ac4b-4a85ad8031fa req-10faed45-d78e-4452-9e70-a51d27ce9373 service nova] Acquiring lock "refresh_cache-ad749135-08c1-48ca-af56-300d6e796012" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 836.419014] env[62499]: DEBUG nova.network.neutron [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 836.519315] env[62499]: DEBUG nova.network.neutron [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 836.694565] env[62499]: DEBUG nova.compute.utils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 836.698278] env[62499]: DEBUG nova.compute.manager [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 836.698278] env[62499]: DEBUG nova.network.neutron [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 836.736675] env[62499]: DEBUG nova.policy [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4de905221d184f00a5579c750ca56325', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd5be96f3d8334da791f2caae93e0624e', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 837.020260] env[62499]: DEBUG nova.network.neutron [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Successfully created port: b7ead464-8575-4153-96fc-fab89ca560f8 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 837.021160] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Releasing lock "refresh_cache-ad749135-08c1-48ca-af56-300d6e796012" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 837.021550] env[62499]: DEBUG nova.compute.manager [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 837.021738] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 837.022061] env[62499]: DEBUG oslo_concurrency.lockutils [req-79efc774-46c7-4351-ac4b-4a85ad8031fa req-10faed45-d78e-4452-9e70-a51d27ce9373 service nova] Acquired lock "refresh_cache-ad749135-08c1-48ca-af56-300d6e796012" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 837.022232] env[62499]: DEBUG nova.network.neutron [req-79efc774-46c7-4351-ac4b-4a85ad8031fa req-10faed45-d78e-4452-9e70-a51d27ce9373 service nova] [instance: ad749135-08c1-48ca-af56-300d6e796012] Refreshing network info cache for port 69123df3-56d1-490c-9303-b6427e432f7f {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 837.023221] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3367c900-de97-47d6-b6ad-247584952b8b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.033550] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7373a65-d38d-4a9d-999c-f6bb856bc2cf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.056069] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ad749135-08c1-48ca-af56-300d6e796012 could not be found. [ 837.056310] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 837.056493] env[62499]: INFO nova.compute.manager [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Took 0.03 seconds to destroy the instance on the hypervisor. [ 837.056735] env[62499]: DEBUG oslo.service.loopingcall [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 837.056993] env[62499]: DEBUG nova.compute.manager [-] [instance: ad749135-08c1-48ca-af56-300d6e796012] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 837.057064] env[62499]: DEBUG nova.network.neutron [-] [instance: ad749135-08c1-48ca-af56-300d6e796012] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 837.071316] env[62499]: DEBUG nova.network.neutron [-] [instance: ad749135-08c1-48ca-af56-300d6e796012] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 837.204060] env[62499]: DEBUG nova.compute.manager [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 837.486151] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f620e7e7-347e-49c9-b6b9-cd818c7febac {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.495507] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b78496c7-cd5c-4805-9242-6d134f1c25b9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.527333] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-461f7a57-d79b-4e58-9937-2d9c9c673df1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.536936] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e408ef88-66d8-4013-b7f4-496cd91e3cdc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.550307] env[62499]: DEBUG nova.compute.provider_tree [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 837.552124] env[62499]: DEBUG nova.network.neutron [req-79efc774-46c7-4351-ac4b-4a85ad8031fa req-10faed45-d78e-4452-9e70-a51d27ce9373 service nova] [instance: ad749135-08c1-48ca-af56-300d6e796012] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 837.574125] env[62499]: DEBUG nova.network.neutron [-] [instance: ad749135-08c1-48ca-af56-300d6e796012] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 837.642707] env[62499]: DEBUG nova.network.neutron [req-79efc774-46c7-4351-ac4b-4a85ad8031fa req-10faed45-d78e-4452-9e70-a51d27ce9373 service nova] [instance: ad749135-08c1-48ca-af56-300d6e796012] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 837.964667] env[62499]: ERROR nova.compute.manager [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b7ead464-8575-4153-96fc-fab89ca560f8, please check neutron logs for more information. [ 837.964667] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 837.964667] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 837.964667] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 837.964667] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 837.964667] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 837.964667] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 837.964667] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 837.964667] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 837.964667] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 837.964667] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 837.964667] env[62499]: ERROR nova.compute.manager raise self.value [ 837.964667] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 837.964667] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 837.964667] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 837.964667] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 837.965312] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 837.965312] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 837.965312] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b7ead464-8575-4153-96fc-fab89ca560f8, please check neutron logs for more information. [ 837.965312] env[62499]: ERROR nova.compute.manager [ 837.965312] env[62499]: Traceback (most recent call last): [ 837.965312] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 837.965312] env[62499]: listener.cb(fileno) [ 837.965312] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 837.965312] env[62499]: result = function(*args, **kwargs) [ 837.965312] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 837.965312] env[62499]: return func(*args, **kwargs) [ 837.965312] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 837.965312] env[62499]: raise e [ 837.965312] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 837.965312] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 837.965312] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 837.965312] env[62499]: created_port_ids = self._update_ports_for_instance( [ 837.965312] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 837.965312] env[62499]: with excutils.save_and_reraise_exception(): [ 837.965312] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 837.965312] env[62499]: self.force_reraise() [ 837.965312] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 837.965312] env[62499]: raise self.value [ 837.965312] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 837.965312] env[62499]: updated_port = self._update_port( [ 837.965312] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 837.965312] env[62499]: _ensure_no_port_binding_failure(port) [ 837.965312] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 837.965312] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 837.966123] env[62499]: nova.exception.PortBindingFailed: Binding failed for port b7ead464-8575-4153-96fc-fab89ca560f8, please check neutron logs for more information. [ 837.966123] env[62499]: Removing descriptor: 16 [ 838.055762] env[62499]: DEBUG nova.scheduler.client.report [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 838.076464] env[62499]: INFO nova.compute.manager [-] [instance: ad749135-08c1-48ca-af56-300d6e796012] Took 1.02 seconds to deallocate network for instance. [ 838.078512] env[62499]: DEBUG nova.compute.claims [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 838.078690] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 838.145558] env[62499]: DEBUG oslo_concurrency.lockutils [req-79efc774-46c7-4351-ac4b-4a85ad8031fa req-10faed45-d78e-4452-9e70-a51d27ce9373 service nova] Releasing lock "refresh_cache-ad749135-08c1-48ca-af56-300d6e796012" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 838.145824] env[62499]: DEBUG nova.compute.manager [req-79efc774-46c7-4351-ac4b-4a85ad8031fa req-10faed45-d78e-4452-9e70-a51d27ce9373 service nova] [instance: ad749135-08c1-48ca-af56-300d6e796012] Received event network-vif-deleted-69123df3-56d1-490c-9303-b6427e432f7f {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 838.213941] env[62499]: DEBUG nova.compute.manager [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 838.241910] env[62499]: DEBUG nova.virt.hardware [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 838.242182] env[62499]: DEBUG nova.virt.hardware [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 838.242338] env[62499]: DEBUG nova.virt.hardware [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 838.242515] env[62499]: DEBUG nova.virt.hardware [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 838.242659] env[62499]: DEBUG nova.virt.hardware [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 838.242808] env[62499]: DEBUG nova.virt.hardware [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 838.243032] env[62499]: DEBUG nova.virt.hardware [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 838.243182] env[62499]: DEBUG nova.virt.hardware [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 838.243346] env[62499]: DEBUG nova.virt.hardware [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 838.243503] env[62499]: DEBUG nova.virt.hardware [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 838.243674] env[62499]: DEBUG nova.virt.hardware [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 838.244537] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-434bfe38-23e4-4452-8218-229c539219fe {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 838.252405] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad45a01e-e179-4f7c-8b76-8c2d9a7c2bdc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 838.265902] env[62499]: ERROR nova.compute.manager [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b7ead464-8575-4153-96fc-fab89ca560f8, please check neutron logs for more information. [ 838.265902] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Traceback (most recent call last): [ 838.265902] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 838.265902] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] yield resources [ 838.265902] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 838.265902] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] self.driver.spawn(context, instance, image_meta, [ 838.265902] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 838.265902] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 838.265902] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 838.265902] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] vm_ref = self.build_virtual_machine(instance, [ 838.265902] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 838.266434] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] vif_infos = vmwarevif.get_vif_info(self._session, [ 838.266434] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 838.266434] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] for vif in network_info: [ 838.266434] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 838.266434] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] return self._sync_wrapper(fn, *args, **kwargs) [ 838.266434] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 838.266434] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] self.wait() [ 838.266434] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 838.266434] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] self[:] = self._gt.wait() [ 838.266434] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 838.266434] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] return self._exit_event.wait() [ 838.266434] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 838.266434] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] current.throw(*self._exc) [ 838.266783] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 838.266783] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] result = function(*args, **kwargs) [ 838.266783] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 838.266783] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] return func(*args, **kwargs) [ 838.266783] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 838.266783] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] raise e [ 838.266783] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 838.266783] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] nwinfo = self.network_api.allocate_for_instance( [ 838.266783] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 838.266783] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] created_port_ids = self._update_ports_for_instance( [ 838.266783] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 838.266783] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] with excutils.save_and_reraise_exception(): [ 838.266783] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 838.267092] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] self.force_reraise() [ 838.267092] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 838.267092] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] raise self.value [ 838.267092] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 838.267092] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] updated_port = self._update_port( [ 838.267092] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 838.267092] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] _ensure_no_port_binding_failure(port) [ 838.267092] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 838.267092] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] raise exception.PortBindingFailed(port_id=port['id']) [ 838.267092] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] nova.exception.PortBindingFailed: Binding failed for port b7ead464-8575-4153-96fc-fab89ca560f8, please check neutron logs for more information. [ 838.267092] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] [ 838.267092] env[62499]: INFO nova.compute.manager [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Terminating instance [ 838.268334] env[62499]: DEBUG oslo_concurrency.lockutils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Acquiring lock "refresh_cache-2a1c52ab-8ee6-4b63-ad04-2450a60262d0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 838.268492] env[62499]: DEBUG oslo_concurrency.lockutils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Acquired lock "refresh_cache-2a1c52ab-8ee6-4b63-ad04-2450a60262d0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 838.268658] env[62499]: DEBUG nova.network.neutron [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 838.372535] env[62499]: DEBUG nova.compute.manager [req-a7a470f5-af21-4e8e-92fe-d5b13f35be2e req-6ffb5cf1-8828-4a61-b442-a82fe9a24469 service nova] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Received event network-changed-b7ead464-8575-4153-96fc-fab89ca560f8 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 838.372690] env[62499]: DEBUG nova.compute.manager [req-a7a470f5-af21-4e8e-92fe-d5b13f35be2e req-6ffb5cf1-8828-4a61-b442-a82fe9a24469 service nova] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Refreshing instance network info cache due to event network-changed-b7ead464-8575-4153-96fc-fab89ca560f8. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 838.372917] env[62499]: DEBUG oslo_concurrency.lockutils [req-a7a470f5-af21-4e8e-92fe-d5b13f35be2e req-6ffb5cf1-8828-4a61-b442-a82fe9a24469 service nova] Acquiring lock "refresh_cache-2a1c52ab-8ee6-4b63-ad04-2450a60262d0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 838.561181] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.372s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 838.561726] env[62499]: DEBUG nova.compute.manager [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 838.564415] env[62499]: DEBUG oslo_concurrency.lockutils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.366s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 838.813561] env[62499]: DEBUG nova.network.neutron [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 838.886056] env[62499]: DEBUG nova.network.neutron [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 839.073394] env[62499]: DEBUG nova.compute.utils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 839.074907] env[62499]: DEBUG nova.compute.manager [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 839.075092] env[62499]: DEBUG nova.network.neutron [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 839.116091] env[62499]: DEBUG nova.policy [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a4763a330d70462182e5bdccc2a08c25', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9512415a4f124e8c9120f6c115f78949', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 839.334031] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a232b6f-e0bd-4546-bb88-3ef7b99a4076 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.344164] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35e84eb7-efd9-40c3-ade2-5b8c1facb939 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.377974] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45293e2e-ba36-4f27-90e7-00c8bf041d0f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.385601] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2cd84356-f289-4d2c-93ab-035647cd8b18 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.389700] env[62499]: DEBUG oslo_concurrency.lockutils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Releasing lock "refresh_cache-2a1c52ab-8ee6-4b63-ad04-2450a60262d0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 839.390173] env[62499]: DEBUG nova.compute.manager [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 839.390360] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 839.392468] env[62499]: DEBUG oslo_concurrency.lockutils [req-a7a470f5-af21-4e8e-92fe-d5b13f35be2e req-6ffb5cf1-8828-4a61-b442-a82fe9a24469 service nova] Acquired lock "refresh_cache-2a1c52ab-8ee6-4b63-ad04-2450a60262d0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 839.392468] env[62499]: DEBUG nova.network.neutron [req-a7a470f5-af21-4e8e-92fe-d5b13f35be2e req-6ffb5cf1-8828-4a61-b442-a82fe9a24469 service nova] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Refreshing network info cache for port b7ead464-8575-4153-96fc-fab89ca560f8 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 839.392468] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-0f50ee9a-0abf-4c87-817a-0d9171b73a32 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.403359] env[62499]: DEBUG nova.compute.provider_tree [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 839.408669] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e071fa0e-1e53-4128-acb4-50ec654b2250 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.419723] env[62499]: DEBUG nova.network.neutron [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Successfully created port: 392f73a3-4972-4a0b-a13a-fef5efe59658 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 839.433322] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 2a1c52ab-8ee6-4b63-ad04-2450a60262d0 could not be found. [ 839.433543] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 839.433722] env[62499]: INFO nova.compute.manager [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Took 0.04 seconds to destroy the instance on the hypervisor. [ 839.433959] env[62499]: DEBUG oslo.service.loopingcall [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 839.434189] env[62499]: DEBUG nova.compute.manager [-] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 839.434282] env[62499]: DEBUG nova.network.neutron [-] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 839.455473] env[62499]: DEBUG nova.network.neutron [-] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 839.578123] env[62499]: DEBUG nova.compute.manager [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 839.907136] env[62499]: DEBUG nova.scheduler.client.report [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 839.910914] env[62499]: DEBUG nova.network.neutron [req-a7a470f5-af21-4e8e-92fe-d5b13f35be2e req-6ffb5cf1-8828-4a61-b442-a82fe9a24469 service nova] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 839.957817] env[62499]: DEBUG nova.network.neutron [-] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 839.986603] env[62499]: DEBUG nova.network.neutron [req-a7a470f5-af21-4e8e-92fe-d5b13f35be2e req-6ffb5cf1-8828-4a61-b442-a82fe9a24469 service nova] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 840.274966] env[62499]: ERROR nova.compute.manager [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 392f73a3-4972-4a0b-a13a-fef5efe59658, please check neutron logs for more information. [ 840.274966] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 840.274966] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 840.274966] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 840.274966] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 840.274966] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 840.274966] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 840.274966] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 840.274966] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 840.274966] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 840.274966] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 840.274966] env[62499]: ERROR nova.compute.manager raise self.value [ 840.274966] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 840.274966] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 840.274966] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 840.274966] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 840.275565] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 840.275565] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 840.275565] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 392f73a3-4972-4a0b-a13a-fef5efe59658, please check neutron logs for more information. [ 840.275565] env[62499]: ERROR nova.compute.manager [ 840.275565] env[62499]: Traceback (most recent call last): [ 840.275565] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 840.275565] env[62499]: listener.cb(fileno) [ 840.275565] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 840.275565] env[62499]: result = function(*args, **kwargs) [ 840.275565] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 840.275565] env[62499]: return func(*args, **kwargs) [ 840.275565] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 840.275565] env[62499]: raise e [ 840.275565] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 840.275565] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 840.275565] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 840.275565] env[62499]: created_port_ids = self._update_ports_for_instance( [ 840.275565] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 840.275565] env[62499]: with excutils.save_and_reraise_exception(): [ 840.275565] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 840.275565] env[62499]: self.force_reraise() [ 840.275565] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 840.275565] env[62499]: raise self.value [ 840.275565] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 840.275565] env[62499]: updated_port = self._update_port( [ 840.275565] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 840.275565] env[62499]: _ensure_no_port_binding_failure(port) [ 840.275565] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 840.275565] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 840.276241] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 392f73a3-4972-4a0b-a13a-fef5efe59658, please check neutron logs for more information. [ 840.276241] env[62499]: Removing descriptor: 16 [ 840.396549] env[62499]: DEBUG nova.compute.manager [req-ef5d3520-769b-4739-a06b-c608142b894a req-32d11d5b-405b-4d17-8b95-b14216537961 service nova] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Received event network-changed-392f73a3-4972-4a0b-a13a-fef5efe59658 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 840.396759] env[62499]: DEBUG nova.compute.manager [req-ef5d3520-769b-4739-a06b-c608142b894a req-32d11d5b-405b-4d17-8b95-b14216537961 service nova] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Refreshing instance network info cache due to event network-changed-392f73a3-4972-4a0b-a13a-fef5efe59658. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 840.396973] env[62499]: DEBUG oslo_concurrency.lockutils [req-ef5d3520-769b-4739-a06b-c608142b894a req-32d11d5b-405b-4d17-8b95-b14216537961 service nova] Acquiring lock "refresh_cache-6da95995-0527-4b05-94dc-a4fcf0ae2f92" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 840.397133] env[62499]: DEBUG oslo_concurrency.lockutils [req-ef5d3520-769b-4739-a06b-c608142b894a req-32d11d5b-405b-4d17-8b95-b14216537961 service nova] Acquired lock "refresh_cache-6da95995-0527-4b05-94dc-a4fcf0ae2f92" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 840.397293] env[62499]: DEBUG nova.network.neutron [req-ef5d3520-769b-4739-a06b-c608142b894a req-32d11d5b-405b-4d17-8b95-b14216537961 service nova] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Refreshing network info cache for port 392f73a3-4972-4a0b-a13a-fef5efe59658 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 840.413784] env[62499]: DEBUG oslo_concurrency.lockutils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.849s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 840.414383] env[62499]: ERROR nova.compute.manager [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1863af9b-2409-4da1-a6dc-d2bb4324e5ec, please check neutron logs for more information. [ 840.414383] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Traceback (most recent call last): [ 840.414383] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 840.414383] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] self.driver.spawn(context, instance, image_meta, [ 840.414383] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 840.414383] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 840.414383] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 840.414383] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] vm_ref = self.build_virtual_machine(instance, [ 840.414383] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 840.414383] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] vif_infos = vmwarevif.get_vif_info(self._session, [ 840.414383] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 840.414739] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] for vif in network_info: [ 840.414739] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 840.414739] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] return self._sync_wrapper(fn, *args, **kwargs) [ 840.414739] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 840.414739] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] self.wait() [ 840.414739] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 840.414739] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] self[:] = self._gt.wait() [ 840.414739] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 840.414739] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] return self._exit_event.wait() [ 840.414739] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 840.414739] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] current.throw(*self._exc) [ 840.414739] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 840.414739] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] result = function(*args, **kwargs) [ 840.415110] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 840.415110] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] return func(*args, **kwargs) [ 840.415110] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 840.415110] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] raise e [ 840.415110] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 840.415110] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] nwinfo = self.network_api.allocate_for_instance( [ 840.415110] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 840.415110] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] created_port_ids = self._update_ports_for_instance( [ 840.415110] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 840.415110] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] with excutils.save_and_reraise_exception(): [ 840.415110] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 840.415110] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] self.force_reraise() [ 840.415110] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 840.415465] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] raise self.value [ 840.415465] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 840.415465] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] updated_port = self._update_port( [ 840.415465] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 840.415465] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] _ensure_no_port_binding_failure(port) [ 840.415465] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 840.415465] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] raise exception.PortBindingFailed(port_id=port['id']) [ 840.415465] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] nova.exception.PortBindingFailed: Binding failed for port 1863af9b-2409-4da1-a6dc-d2bb4324e5ec, please check neutron logs for more information. [ 840.415465] env[62499]: ERROR nova.compute.manager [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] [ 840.415465] env[62499]: DEBUG nova.compute.utils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Binding failed for port 1863af9b-2409-4da1-a6dc-d2bb4324e5ec, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 840.416328] env[62499]: DEBUG oslo_concurrency.lockutils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.382s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 840.418066] env[62499]: INFO nova.compute.claims [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 840.420935] env[62499]: DEBUG nova.compute.manager [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Build of instance 261cbefa-fb2d-48da-a4fe-80b744a931f6 was re-scheduled: Binding failed for port 1863af9b-2409-4da1-a6dc-d2bb4324e5ec, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 840.421477] env[62499]: DEBUG nova.compute.manager [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 840.421696] env[62499]: DEBUG oslo_concurrency.lockutils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Acquiring lock "refresh_cache-261cbefa-fb2d-48da-a4fe-80b744a931f6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 840.421843] env[62499]: DEBUG oslo_concurrency.lockutils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Acquired lock "refresh_cache-261cbefa-fb2d-48da-a4fe-80b744a931f6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 840.422058] env[62499]: DEBUG nova.network.neutron [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 840.461493] env[62499]: INFO nova.compute.manager [-] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Took 1.03 seconds to deallocate network for instance. [ 840.462863] env[62499]: DEBUG nova.compute.claims [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 840.463071] env[62499]: DEBUG oslo_concurrency.lockutils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 840.488662] env[62499]: DEBUG oslo_concurrency.lockutils [req-a7a470f5-af21-4e8e-92fe-d5b13f35be2e req-6ffb5cf1-8828-4a61-b442-a82fe9a24469 service nova] Releasing lock "refresh_cache-2a1c52ab-8ee6-4b63-ad04-2450a60262d0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 840.488946] env[62499]: DEBUG nova.compute.manager [req-a7a470f5-af21-4e8e-92fe-d5b13f35be2e req-6ffb5cf1-8828-4a61-b442-a82fe9a24469 service nova] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Received event network-vif-deleted-b7ead464-8575-4153-96fc-fab89ca560f8 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 840.593265] env[62499]: DEBUG nova.compute.manager [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 840.617305] env[62499]: DEBUG nova.virt.hardware [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 840.617547] env[62499]: DEBUG nova.virt.hardware [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 840.617706] env[62499]: DEBUG nova.virt.hardware [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 840.617920] env[62499]: DEBUG nova.virt.hardware [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 840.618091] env[62499]: DEBUG nova.virt.hardware [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 840.618246] env[62499]: DEBUG nova.virt.hardware [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 840.618447] env[62499]: DEBUG nova.virt.hardware [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 840.618605] env[62499]: DEBUG nova.virt.hardware [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 840.618770] env[62499]: DEBUG nova.virt.hardware [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 840.618927] env[62499]: DEBUG nova.virt.hardware [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 840.619122] env[62499]: DEBUG nova.virt.hardware [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 840.620025] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ea1e000-c2ce-4a91-a46a-97606c24fd42 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 840.628557] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c31b635d-e4f5-4ebd-ae58-c02289cc7040 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 840.641966] env[62499]: ERROR nova.compute.manager [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 392f73a3-4972-4a0b-a13a-fef5efe59658, please check neutron logs for more information. [ 840.641966] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Traceback (most recent call last): [ 840.641966] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 840.641966] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] yield resources [ 840.641966] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 840.641966] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] self.driver.spawn(context, instance, image_meta, [ 840.641966] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 840.641966] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] self._vmops.spawn(context, instance, image_meta, injected_files, [ 840.641966] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 840.641966] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] vm_ref = self.build_virtual_machine(instance, [ 840.641966] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 840.642354] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] vif_infos = vmwarevif.get_vif_info(self._session, [ 840.642354] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 840.642354] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] for vif in network_info: [ 840.642354] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 840.642354] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] return self._sync_wrapper(fn, *args, **kwargs) [ 840.642354] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 840.642354] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] self.wait() [ 840.642354] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 840.642354] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] self[:] = self._gt.wait() [ 840.642354] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 840.642354] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] return self._exit_event.wait() [ 840.642354] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 840.642354] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] current.throw(*self._exc) [ 840.642768] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 840.642768] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] result = function(*args, **kwargs) [ 840.642768] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 840.642768] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] return func(*args, **kwargs) [ 840.642768] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 840.642768] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] raise e [ 840.642768] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 840.642768] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] nwinfo = self.network_api.allocate_for_instance( [ 840.642768] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 840.642768] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] created_port_ids = self._update_ports_for_instance( [ 840.642768] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 840.642768] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] with excutils.save_and_reraise_exception(): [ 840.642768] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 840.643136] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] self.force_reraise() [ 840.643136] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 840.643136] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] raise self.value [ 840.643136] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 840.643136] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] updated_port = self._update_port( [ 840.643136] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 840.643136] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] _ensure_no_port_binding_failure(port) [ 840.643136] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 840.643136] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] raise exception.PortBindingFailed(port_id=port['id']) [ 840.643136] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] nova.exception.PortBindingFailed: Binding failed for port 392f73a3-4972-4a0b-a13a-fef5efe59658, please check neutron logs for more information. [ 840.643136] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] [ 840.643136] env[62499]: INFO nova.compute.manager [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Terminating instance [ 840.644436] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquiring lock "refresh_cache-6da95995-0527-4b05-94dc-a4fcf0ae2f92" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 840.917522] env[62499]: DEBUG nova.network.neutron [req-ef5d3520-769b-4739-a06b-c608142b894a req-32d11d5b-405b-4d17-8b95-b14216537961 service nova] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 840.939616] env[62499]: DEBUG nova.network.neutron [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 840.982831] env[62499]: DEBUG nova.network.neutron [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 840.997286] env[62499]: DEBUG nova.network.neutron [req-ef5d3520-769b-4739-a06b-c608142b894a req-32d11d5b-405b-4d17-8b95-b14216537961 service nova] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 841.485481] env[62499]: DEBUG oslo_concurrency.lockutils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Releasing lock "refresh_cache-261cbefa-fb2d-48da-a4fe-80b744a931f6" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 841.485772] env[62499]: DEBUG nova.compute.manager [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 841.486023] env[62499]: DEBUG nova.compute.manager [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 841.486237] env[62499]: DEBUG nova.network.neutron [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 841.500899] env[62499]: DEBUG oslo_concurrency.lockutils [req-ef5d3520-769b-4739-a06b-c608142b894a req-32d11d5b-405b-4d17-8b95-b14216537961 service nova] Releasing lock "refresh_cache-6da95995-0527-4b05-94dc-a4fcf0ae2f92" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 841.501180] env[62499]: DEBUG nova.compute.manager [req-ef5d3520-769b-4739-a06b-c608142b894a req-32d11d5b-405b-4d17-8b95-b14216537961 service nova] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Received event network-vif-deleted-392f73a3-4972-4a0b-a13a-fef5efe59658 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 841.502031] env[62499]: DEBUG nova.network.neutron [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 841.503707] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquired lock "refresh_cache-6da95995-0527-4b05-94dc-a4fcf0ae2f92" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 841.503891] env[62499]: DEBUG nova.network.neutron [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 841.689248] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be0a95d5-abf4-4ccd-a480-da6865760c3c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 841.697159] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c05f6a0d-ef61-4d01-8249-6fce2a465007 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 841.728325] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-072e3d40-a20e-48d1-b9b8-0948dc36461e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 841.735721] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19be1456-5706-4bb6-b14f-46d58fb4c91e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 841.749156] env[62499]: DEBUG nova.compute.provider_tree [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 842.004996] env[62499]: DEBUG nova.network.neutron [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 842.155300] env[62499]: DEBUG nova.network.neutron [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 842.222083] env[62499]: DEBUG nova.network.neutron [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 842.252290] env[62499]: DEBUG nova.scheduler.client.report [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 842.507310] env[62499]: INFO nova.compute.manager [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 261cbefa-fb2d-48da-a4fe-80b744a931f6] Took 1.02 seconds to deallocate network for instance. [ 842.725043] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Releasing lock "refresh_cache-6da95995-0527-4b05-94dc-a4fcf0ae2f92" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 842.725043] env[62499]: DEBUG nova.compute.manager [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 842.725198] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 842.725401] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5f587adc-1303-4ae6-9def-67a3c8ea57e9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 842.735016] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32a14336-dd55-461d-9bfc-ef0279243b27 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 842.757959] env[62499]: DEBUG oslo_concurrency.lockutils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.342s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 842.758470] env[62499]: DEBUG nova.compute.manager [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 842.761795] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6da95995-0527-4b05-94dc-a4fcf0ae2f92 could not be found. [ 842.761795] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 842.761795] env[62499]: INFO nova.compute.manager [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Took 0.04 seconds to destroy the instance on the hypervisor. [ 842.761908] env[62499]: DEBUG oslo.service.loopingcall [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 842.762323] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.003s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 842.763666] env[62499]: INFO nova.compute.claims [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 842.766060] env[62499]: DEBUG nova.compute.manager [-] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 842.766200] env[62499]: DEBUG nova.network.neutron [-] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 842.841615] env[62499]: DEBUG nova.network.neutron [-] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 843.264413] env[62499]: DEBUG nova.compute.utils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 843.265748] env[62499]: DEBUG nova.compute.manager [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 843.265922] env[62499]: DEBUG nova.network.neutron [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 843.317982] env[62499]: DEBUG nova.policy [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4762545dc5e14c16b5cbb1bad27d90a2', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3f11318c300c4c308f3f8a14db355954', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 843.344315] env[62499]: DEBUG nova.network.neutron [-] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 843.536559] env[62499]: INFO nova.scheduler.client.report [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Deleted allocations for instance 261cbefa-fb2d-48da-a4fe-80b744a931f6 [ 843.602218] env[62499]: DEBUG nova.network.neutron [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Successfully created port: f0eb55e9-0018-4c0e-99cb-227551afe2c8 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 843.769748] env[62499]: DEBUG nova.compute.manager [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 843.846629] env[62499]: INFO nova.compute.manager [-] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Took 1.08 seconds to deallocate network for instance. [ 843.849254] env[62499]: DEBUG nova.compute.claims [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 843.849437] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 844.044418] env[62499]: DEBUG oslo_concurrency.lockutils [None req-86297533-7bbc-4aeb-be79-5ff5e2437213 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Lock "261cbefa-fb2d-48da-a4fe-80b744a931f6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 143.049s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 844.051676] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-68d3c8ca-4224-4328-b78b-3f52d637d181 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 844.059887] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cda77513-aaa4-4707-8916-2169935b5858 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 844.090647] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-124583ec-a48d-4a75-884b-c65538adc629 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 844.097653] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-465dcea9-9cb9-4558-b1ed-1cb875e2ab53 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 844.111378] env[62499]: DEBUG nova.compute.provider_tree [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 844.348474] env[62499]: DEBUG nova.compute.manager [req-7ae3993d-e8f4-4049-8ff1-41fa77e5c482 req-fa59f0b1-266e-4c5e-922d-06190825f7f0 service nova] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Received event network-changed-f0eb55e9-0018-4c0e-99cb-227551afe2c8 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 844.348685] env[62499]: DEBUG nova.compute.manager [req-7ae3993d-e8f4-4049-8ff1-41fa77e5c482 req-fa59f0b1-266e-4c5e-922d-06190825f7f0 service nova] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Refreshing instance network info cache due to event network-changed-f0eb55e9-0018-4c0e-99cb-227551afe2c8. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 844.349155] env[62499]: DEBUG oslo_concurrency.lockutils [req-7ae3993d-e8f4-4049-8ff1-41fa77e5c482 req-fa59f0b1-266e-4c5e-922d-06190825f7f0 service nova] Acquiring lock "refresh_cache-263931cd-b2dc-41bb-8a2b-abf61aadafc9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 844.349155] env[62499]: DEBUG oslo_concurrency.lockutils [req-7ae3993d-e8f4-4049-8ff1-41fa77e5c482 req-fa59f0b1-266e-4c5e-922d-06190825f7f0 service nova] Acquired lock "refresh_cache-263931cd-b2dc-41bb-8a2b-abf61aadafc9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 844.349236] env[62499]: DEBUG nova.network.neutron [req-7ae3993d-e8f4-4049-8ff1-41fa77e5c482 req-fa59f0b1-266e-4c5e-922d-06190825f7f0 service nova] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Refreshing network info cache for port f0eb55e9-0018-4c0e-99cb-227551afe2c8 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 844.517659] env[62499]: ERROR nova.compute.manager [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f0eb55e9-0018-4c0e-99cb-227551afe2c8, please check neutron logs for more information. [ 844.517659] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 844.517659] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 844.517659] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 844.517659] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 844.517659] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 844.517659] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 844.517659] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 844.517659] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 844.517659] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 844.517659] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 844.517659] env[62499]: ERROR nova.compute.manager raise self.value [ 844.517659] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 844.517659] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 844.517659] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 844.517659] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 844.518284] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 844.518284] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 844.518284] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f0eb55e9-0018-4c0e-99cb-227551afe2c8, please check neutron logs for more information. [ 844.518284] env[62499]: ERROR nova.compute.manager [ 844.518284] env[62499]: Traceback (most recent call last): [ 844.518284] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 844.518284] env[62499]: listener.cb(fileno) [ 844.518284] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 844.518284] env[62499]: result = function(*args, **kwargs) [ 844.518284] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 844.518284] env[62499]: return func(*args, **kwargs) [ 844.518284] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 844.518284] env[62499]: raise e [ 844.518284] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 844.518284] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 844.518284] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 844.518284] env[62499]: created_port_ids = self._update_ports_for_instance( [ 844.518284] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 844.518284] env[62499]: with excutils.save_and_reraise_exception(): [ 844.518284] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 844.518284] env[62499]: self.force_reraise() [ 844.518284] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 844.518284] env[62499]: raise self.value [ 844.518284] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 844.518284] env[62499]: updated_port = self._update_port( [ 844.518284] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 844.518284] env[62499]: _ensure_no_port_binding_failure(port) [ 844.518284] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 844.518284] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 844.519270] env[62499]: nova.exception.PortBindingFailed: Binding failed for port f0eb55e9-0018-4c0e-99cb-227551afe2c8, please check neutron logs for more information. [ 844.519270] env[62499]: Removing descriptor: 16 [ 844.548554] env[62499]: DEBUG nova.compute.manager [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 844.614108] env[62499]: DEBUG nova.scheduler.client.report [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 844.783609] env[62499]: DEBUG nova.compute.manager [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 844.808101] env[62499]: DEBUG nova.virt.hardware [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 844.808352] env[62499]: DEBUG nova.virt.hardware [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 844.808507] env[62499]: DEBUG nova.virt.hardware [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 844.808683] env[62499]: DEBUG nova.virt.hardware [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 844.808992] env[62499]: DEBUG nova.virt.hardware [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 844.809181] env[62499]: DEBUG nova.virt.hardware [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 844.809380] env[62499]: DEBUG nova.virt.hardware [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 844.809536] env[62499]: DEBUG nova.virt.hardware [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 844.809699] env[62499]: DEBUG nova.virt.hardware [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 844.809909] env[62499]: DEBUG nova.virt.hardware [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 844.810111] env[62499]: DEBUG nova.virt.hardware [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 844.810967] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f299bcdd-50b8-4288-a586-f1941f83d456 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 844.818666] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b72daa4c-dc4d-49a6-bc1a-5fcec11df356 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 844.832524] env[62499]: ERROR nova.compute.manager [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f0eb55e9-0018-4c0e-99cb-227551afe2c8, please check neutron logs for more information. [ 844.832524] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Traceback (most recent call last): [ 844.832524] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 844.832524] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] yield resources [ 844.832524] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 844.832524] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] self.driver.spawn(context, instance, image_meta, [ 844.832524] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 844.832524] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 844.832524] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 844.832524] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] vm_ref = self.build_virtual_machine(instance, [ 844.832524] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 844.833105] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] vif_infos = vmwarevif.get_vif_info(self._session, [ 844.833105] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 844.833105] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] for vif in network_info: [ 844.833105] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 844.833105] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] return self._sync_wrapper(fn, *args, **kwargs) [ 844.833105] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 844.833105] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] self.wait() [ 844.833105] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 844.833105] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] self[:] = self._gt.wait() [ 844.833105] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 844.833105] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] return self._exit_event.wait() [ 844.833105] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 844.833105] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] current.throw(*self._exc) [ 844.833576] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 844.833576] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] result = function(*args, **kwargs) [ 844.833576] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 844.833576] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] return func(*args, **kwargs) [ 844.833576] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 844.833576] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] raise e [ 844.833576] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 844.833576] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] nwinfo = self.network_api.allocate_for_instance( [ 844.833576] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 844.833576] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] created_port_ids = self._update_ports_for_instance( [ 844.833576] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 844.833576] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] with excutils.save_and_reraise_exception(): [ 844.833576] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 844.833965] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] self.force_reraise() [ 844.833965] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 844.833965] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] raise self.value [ 844.833965] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 844.833965] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] updated_port = self._update_port( [ 844.833965] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 844.833965] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] _ensure_no_port_binding_failure(port) [ 844.833965] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 844.833965] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] raise exception.PortBindingFailed(port_id=port['id']) [ 844.833965] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] nova.exception.PortBindingFailed: Binding failed for port f0eb55e9-0018-4c0e-99cb-227551afe2c8, please check neutron logs for more information. [ 844.833965] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] [ 844.833965] env[62499]: INFO nova.compute.manager [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Terminating instance [ 844.835109] env[62499]: DEBUG oslo_concurrency.lockutils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "refresh_cache-263931cd-b2dc-41bb-8a2b-abf61aadafc9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 844.870716] env[62499]: DEBUG nova.network.neutron [req-7ae3993d-e8f4-4049-8ff1-41fa77e5c482 req-fa59f0b1-266e-4c5e-922d-06190825f7f0 service nova] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 844.960955] env[62499]: DEBUG nova.network.neutron [req-7ae3993d-e8f4-4049-8ff1-41fa77e5c482 req-fa59f0b1-266e-4c5e-922d-06190825f7f0 service nova] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 845.072316] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 845.118858] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.356s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 845.119415] env[62499]: DEBUG nova.compute.manager [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 845.122583] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 18.308s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 845.123308] env[62499]: DEBUG nova.objects.instance [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 39bc8b3a-e9d9-48e3-a193-5fbdc2454346] Trying to apply a migration context that does not seem to be set for this instance {{(pid=62499) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 845.463730] env[62499]: DEBUG oslo_concurrency.lockutils [req-7ae3993d-e8f4-4049-8ff1-41fa77e5c482 req-fa59f0b1-266e-4c5e-922d-06190825f7f0 service nova] Releasing lock "refresh_cache-263931cd-b2dc-41bb-8a2b-abf61aadafc9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 845.463894] env[62499]: DEBUG oslo_concurrency.lockutils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquired lock "refresh_cache-263931cd-b2dc-41bb-8a2b-abf61aadafc9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 845.464108] env[62499]: DEBUG nova.network.neutron [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 845.629753] env[62499]: DEBUG nova.compute.utils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 845.633784] env[62499]: DEBUG nova.compute.manager [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 845.633954] env[62499]: DEBUG nova.network.neutron [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 845.671279] env[62499]: DEBUG nova.policy [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '108621ddd4b2490db4e763b24a9dca73', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '2fa336a944de4078b3a0ef7578f556d0', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 845.961699] env[62499]: DEBUG nova.network.neutron [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Successfully created port: 588d004d-9a19-4de8-9e31-22dc0aeede5b {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 845.991741] env[62499]: DEBUG nova.network.neutron [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 846.068818] env[62499]: DEBUG nova.network.neutron [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 846.134668] env[62499]: DEBUG nova.compute.manager [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 846.138161] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9cebef17-ae3b-41bf-bdeb-f097967f0c03 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.016s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 846.140738] env[62499]: DEBUG oslo_concurrency.lockutils [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 18.175s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 846.140738] env[62499]: DEBUG nova.objects.instance [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] [instance: 15f8e04f-8c14-4315-8686-da4db517f7b6] Trying to apply a migration context that does not seem to be set for this instance {{(pid=62499) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 846.373900] env[62499]: DEBUG nova.compute.manager [req-c24c970b-cd25-4982-abed-4e21c029aa15 req-632df227-80fb-449f-8318-9afe264b08df service nova] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Received event network-vif-deleted-f0eb55e9-0018-4c0e-99cb-227551afe2c8 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 846.575238] env[62499]: DEBUG oslo_concurrency.lockutils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Releasing lock "refresh_cache-263931cd-b2dc-41bb-8a2b-abf61aadafc9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 846.575687] env[62499]: DEBUG nova.compute.manager [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 846.575878] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 846.576202] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c301d51a-8596-4b0c-981a-8b0f05776cee {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 846.585118] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9517c617-4e2b-4261-bd70-54062eb4c256 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 846.606339] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 263931cd-b2dc-41bb-8a2b-abf61aadafc9 could not be found. [ 846.606887] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 846.606887] env[62499]: INFO nova.compute.manager [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Took 0.03 seconds to destroy the instance on the hypervisor. [ 846.607065] env[62499]: DEBUG oslo.service.loopingcall [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 846.608909] env[62499]: DEBUG nova.compute.manager [-] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 846.609017] env[62499]: DEBUG nova.network.neutron [-] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 846.643786] env[62499]: DEBUG nova.network.neutron [-] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 846.809194] env[62499]: ERROR nova.compute.manager [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 588d004d-9a19-4de8-9e31-22dc0aeede5b, please check neutron logs for more information. [ 846.809194] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 846.809194] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 846.809194] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 846.809194] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 846.809194] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 846.809194] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 846.809194] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 846.809194] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 846.809194] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 846.809194] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 846.809194] env[62499]: ERROR nova.compute.manager raise self.value [ 846.809194] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 846.809194] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 846.809194] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 846.809194] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 846.809617] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 846.809617] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 846.809617] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 588d004d-9a19-4de8-9e31-22dc0aeede5b, please check neutron logs for more information. [ 846.809617] env[62499]: ERROR nova.compute.manager [ 846.809617] env[62499]: Traceback (most recent call last): [ 846.809617] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 846.809617] env[62499]: listener.cb(fileno) [ 846.809617] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 846.809617] env[62499]: result = function(*args, **kwargs) [ 846.809617] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 846.809617] env[62499]: return func(*args, **kwargs) [ 846.809617] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 846.809617] env[62499]: raise e [ 846.809617] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 846.809617] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 846.809617] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 846.809617] env[62499]: created_port_ids = self._update_ports_for_instance( [ 846.809617] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 846.809617] env[62499]: with excutils.save_and_reraise_exception(): [ 846.809617] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 846.809617] env[62499]: self.force_reraise() [ 846.809617] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 846.809617] env[62499]: raise self.value [ 846.809617] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 846.809617] env[62499]: updated_port = self._update_port( [ 846.809617] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 846.809617] env[62499]: _ensure_no_port_binding_failure(port) [ 846.809617] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 846.809617] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 846.810545] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 588d004d-9a19-4de8-9e31-22dc0aeede5b, please check neutron logs for more information. [ 846.810545] env[62499]: Removing descriptor: 16 [ 847.151718] env[62499]: DEBUG nova.compute.manager [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 847.153853] env[62499]: DEBUG nova.network.neutron [-] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 847.155945] env[62499]: DEBUG oslo_concurrency.lockutils [None req-72fbff57-f32c-4a3d-8870-46f30b364e69 tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.016s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 847.157261] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 18.447s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 847.158600] env[62499]: INFO nova.compute.claims [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 847.181775] env[62499]: DEBUG nova.virt.hardware [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 847.182024] env[62499]: DEBUG nova.virt.hardware [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 847.182213] env[62499]: DEBUG nova.virt.hardware [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 847.182423] env[62499]: DEBUG nova.virt.hardware [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 847.182577] env[62499]: DEBUG nova.virt.hardware [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 847.182724] env[62499]: DEBUG nova.virt.hardware [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 847.182953] env[62499]: DEBUG nova.virt.hardware [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 847.183144] env[62499]: DEBUG nova.virt.hardware [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 847.183317] env[62499]: DEBUG nova.virt.hardware [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 847.183482] env[62499]: DEBUG nova.virt.hardware [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 847.183659] env[62499]: DEBUG nova.virt.hardware [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 847.184709] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ac02c22-5a00-418f-8483-15ffb02c1f00 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 847.193793] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2b68cab-a86e-4829-81b5-fa5da60eaafe {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 847.207602] env[62499]: ERROR nova.compute.manager [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 588d004d-9a19-4de8-9e31-22dc0aeede5b, please check neutron logs for more information. [ 847.207602] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Traceback (most recent call last): [ 847.207602] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 847.207602] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] yield resources [ 847.207602] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 847.207602] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] self.driver.spawn(context, instance, image_meta, [ 847.207602] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 847.207602] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] self._vmops.spawn(context, instance, image_meta, injected_files, [ 847.207602] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 847.207602] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] vm_ref = self.build_virtual_machine(instance, [ 847.207602] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 847.208096] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] vif_infos = vmwarevif.get_vif_info(self._session, [ 847.208096] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 847.208096] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] for vif in network_info: [ 847.208096] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 847.208096] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] return self._sync_wrapper(fn, *args, **kwargs) [ 847.208096] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 847.208096] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] self.wait() [ 847.208096] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 847.208096] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] self[:] = self._gt.wait() [ 847.208096] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 847.208096] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] return self._exit_event.wait() [ 847.208096] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 847.208096] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] current.throw(*self._exc) [ 847.208473] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 847.208473] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] result = function(*args, **kwargs) [ 847.208473] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 847.208473] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] return func(*args, **kwargs) [ 847.208473] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 847.208473] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] raise e [ 847.208473] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 847.208473] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] nwinfo = self.network_api.allocate_for_instance( [ 847.208473] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 847.208473] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] created_port_ids = self._update_ports_for_instance( [ 847.208473] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 847.208473] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] with excutils.save_and_reraise_exception(): [ 847.208473] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 847.208907] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] self.force_reraise() [ 847.208907] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 847.208907] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] raise self.value [ 847.208907] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 847.208907] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] updated_port = self._update_port( [ 847.208907] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 847.208907] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] _ensure_no_port_binding_failure(port) [ 847.208907] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 847.208907] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] raise exception.PortBindingFailed(port_id=port['id']) [ 847.208907] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] nova.exception.PortBindingFailed: Binding failed for port 588d004d-9a19-4de8-9e31-22dc0aeede5b, please check neutron logs for more information. [ 847.208907] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] [ 847.208907] env[62499]: INFO nova.compute.manager [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Terminating instance [ 847.210440] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Acquiring lock "refresh_cache-20e4ff1b-0b84-477d-a0d0-a85a439449ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 847.210605] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Acquired lock "refresh_cache-20e4ff1b-0b84-477d-a0d0-a85a439449ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 847.210772] env[62499]: DEBUG nova.network.neutron [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 847.658087] env[62499]: INFO nova.compute.manager [-] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Took 1.05 seconds to deallocate network for instance. [ 847.660558] env[62499]: DEBUG nova.compute.claims [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 847.660748] env[62499]: DEBUG oslo_concurrency.lockutils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 847.730770] env[62499]: DEBUG nova.network.neutron [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 847.814213] env[62499]: DEBUG nova.network.neutron [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 848.318384] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Releasing lock "refresh_cache-20e4ff1b-0b84-477d-a0d0-a85a439449ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 848.318875] env[62499]: DEBUG nova.compute.manager [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 848.319181] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 848.319780] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-14ab8806-6d62-4f3c-b7fa-d2991ff38be2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.328396] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb4f534a-e238-4f90-866e-ad78cb1ccbdf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.351535] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 20e4ff1b-0b84-477d-a0d0-a85a439449ad could not be found. [ 848.351747] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 848.351927] env[62499]: INFO nova.compute.manager [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Took 0.03 seconds to destroy the instance on the hypervisor. [ 848.352181] env[62499]: DEBUG oslo.service.loopingcall [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 848.354305] env[62499]: DEBUG nova.compute.manager [-] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 848.354388] env[62499]: DEBUG nova.network.neutron [-] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 848.368778] env[62499]: DEBUG nova.network.neutron [-] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 848.397616] env[62499]: DEBUG nova.compute.manager [req-b458c709-e996-44ef-8705-5c8fa7ea33e5 req-ea4c4554-73b1-4a08-a3a2-9e68eb198303 service nova] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Received event network-changed-588d004d-9a19-4de8-9e31-22dc0aeede5b {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 848.397826] env[62499]: DEBUG nova.compute.manager [req-b458c709-e996-44ef-8705-5c8fa7ea33e5 req-ea4c4554-73b1-4a08-a3a2-9e68eb198303 service nova] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Refreshing instance network info cache due to event network-changed-588d004d-9a19-4de8-9e31-22dc0aeede5b. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 848.398060] env[62499]: DEBUG oslo_concurrency.lockutils [req-b458c709-e996-44ef-8705-5c8fa7ea33e5 req-ea4c4554-73b1-4a08-a3a2-9e68eb198303 service nova] Acquiring lock "refresh_cache-20e4ff1b-0b84-477d-a0d0-a85a439449ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 848.398204] env[62499]: DEBUG oslo_concurrency.lockutils [req-b458c709-e996-44ef-8705-5c8fa7ea33e5 req-ea4c4554-73b1-4a08-a3a2-9e68eb198303 service nova] Acquired lock "refresh_cache-20e4ff1b-0b84-477d-a0d0-a85a439449ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 848.398727] env[62499]: DEBUG nova.network.neutron [req-b458c709-e996-44ef-8705-5c8fa7ea33e5 req-ea4c4554-73b1-4a08-a3a2-9e68eb198303 service nova] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Refreshing network info cache for port 588d004d-9a19-4de8-9e31-22dc0aeede5b {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 848.426718] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e10ceac7-df1d-490e-bda0-909b07c0c202 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.434423] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2093d58b-1302-4fb9-9694-ce344a2de45f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.465885] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd5ee670-34e1-48c5-bd23-3f0b645b7606 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.473828] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e252f0fd-9b5a-40e3-906c-3d377f5aed7a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.487971] env[62499]: DEBUG nova.compute.provider_tree [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 848.871435] env[62499]: DEBUG nova.network.neutron [-] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 848.923346] env[62499]: DEBUG nova.network.neutron [req-b458c709-e996-44ef-8705-5c8fa7ea33e5 req-ea4c4554-73b1-4a08-a3a2-9e68eb198303 service nova] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 848.991098] env[62499]: DEBUG nova.scheduler.client.report [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 849.008162] env[62499]: DEBUG nova.network.neutron [req-b458c709-e996-44ef-8705-5c8fa7ea33e5 req-ea4c4554-73b1-4a08-a3a2-9e68eb198303 service nova] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 849.373932] env[62499]: INFO nova.compute.manager [-] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Took 1.02 seconds to deallocate network for instance. [ 849.376264] env[62499]: DEBUG nova.compute.claims [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 849.376449] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 849.495982] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.339s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 849.496520] env[62499]: DEBUG nova.compute.manager [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 849.499259] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 19.896s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 849.499495] env[62499]: DEBUG nova.objects.instance [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Lazy-loading 'resources' on Instance uuid 57d21761-431b-4a41-86eb-038f8c35d8ef {{(pid=62499) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 849.510085] env[62499]: DEBUG oslo_concurrency.lockutils [req-b458c709-e996-44ef-8705-5c8fa7ea33e5 req-ea4c4554-73b1-4a08-a3a2-9e68eb198303 service nova] Releasing lock "refresh_cache-20e4ff1b-0b84-477d-a0d0-a85a439449ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 849.510315] env[62499]: DEBUG nova.compute.manager [req-b458c709-e996-44ef-8705-5c8fa7ea33e5 req-ea4c4554-73b1-4a08-a3a2-9e68eb198303 service nova] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Received event network-vif-deleted-588d004d-9a19-4de8-9e31-22dc0aeede5b {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 850.002978] env[62499]: DEBUG nova.compute.utils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 850.003987] env[62499]: DEBUG nova.compute.manager [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 850.004183] env[62499]: DEBUG nova.network.neutron [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 850.044679] env[62499]: DEBUG nova.policy [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ffc458db78a04358af903d6119c5eac9', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '967135c837004321824783716458da8e', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 850.259169] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-606c10d6-b1eb-416f-a3b3-f68814ce8b0a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.266864] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af22b648-0017-4d6c-a9f1-de5ca4adcd7f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.298406] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-780a6332-76f6-4983-bf64-456f6c5121ae {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.305750] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3dbd7ac8-6730-40f0-a1d5-f9fdb807e9b8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.318864] env[62499]: DEBUG nova.compute.provider_tree [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 850.365450] env[62499]: DEBUG nova.network.neutron [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Successfully created port: dee6d343-e56a-4d3a-9f5f-2a6fe259825e {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 850.509547] env[62499]: DEBUG nova.compute.manager [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 850.825022] env[62499]: DEBUG nova.scheduler.client.report [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 851.026305] env[62499]: DEBUG nova.compute.manager [req-97b84cb3-5a7f-4ed4-bf76-14d90bdfbe80 req-013ba8f0-626e-4741-b18c-99e372baf6f1 service nova] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Received event network-changed-dee6d343-e56a-4d3a-9f5f-2a6fe259825e {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 851.026581] env[62499]: DEBUG nova.compute.manager [req-97b84cb3-5a7f-4ed4-bf76-14d90bdfbe80 req-013ba8f0-626e-4741-b18c-99e372baf6f1 service nova] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Refreshing instance network info cache due to event network-changed-dee6d343-e56a-4d3a-9f5f-2a6fe259825e. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 851.026810] env[62499]: DEBUG oslo_concurrency.lockutils [req-97b84cb3-5a7f-4ed4-bf76-14d90bdfbe80 req-013ba8f0-626e-4741-b18c-99e372baf6f1 service nova] Acquiring lock "refresh_cache-4eda80db-5db7-4fd6-9932-108c22f4a616" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 851.026945] env[62499]: DEBUG oslo_concurrency.lockutils [req-97b84cb3-5a7f-4ed4-bf76-14d90bdfbe80 req-013ba8f0-626e-4741-b18c-99e372baf6f1 service nova] Acquired lock "refresh_cache-4eda80db-5db7-4fd6-9932-108c22f4a616" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 851.027125] env[62499]: DEBUG nova.network.neutron [req-97b84cb3-5a7f-4ed4-bf76-14d90bdfbe80 req-013ba8f0-626e-4741-b18c-99e372baf6f1 service nova] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Refreshing network info cache for port dee6d343-e56a-4d3a-9f5f-2a6fe259825e {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 851.248887] env[62499]: ERROR nova.compute.manager [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port dee6d343-e56a-4d3a-9f5f-2a6fe259825e, please check neutron logs for more information. [ 851.248887] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 851.248887] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 851.248887] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 851.248887] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 851.248887] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 851.248887] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 851.248887] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 851.248887] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 851.248887] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 851.248887] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 851.248887] env[62499]: ERROR nova.compute.manager raise self.value [ 851.248887] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 851.248887] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 851.248887] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 851.248887] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 851.249392] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 851.249392] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 851.249392] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port dee6d343-e56a-4d3a-9f5f-2a6fe259825e, please check neutron logs for more information. [ 851.249392] env[62499]: ERROR nova.compute.manager [ 851.249392] env[62499]: Traceback (most recent call last): [ 851.249392] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 851.249392] env[62499]: listener.cb(fileno) [ 851.249392] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 851.249392] env[62499]: result = function(*args, **kwargs) [ 851.249392] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 851.249392] env[62499]: return func(*args, **kwargs) [ 851.249392] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 851.249392] env[62499]: raise e [ 851.249392] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 851.249392] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 851.249392] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 851.249392] env[62499]: created_port_ids = self._update_ports_for_instance( [ 851.249392] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 851.249392] env[62499]: with excutils.save_and_reraise_exception(): [ 851.249392] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 851.249392] env[62499]: self.force_reraise() [ 851.249392] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 851.249392] env[62499]: raise self.value [ 851.249392] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 851.249392] env[62499]: updated_port = self._update_port( [ 851.249392] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 851.249392] env[62499]: _ensure_no_port_binding_failure(port) [ 851.249392] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 851.249392] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 851.250157] env[62499]: nova.exception.PortBindingFailed: Binding failed for port dee6d343-e56a-4d3a-9f5f-2a6fe259825e, please check neutron logs for more information. [ 851.250157] env[62499]: Removing descriptor: 16 [ 851.327864] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.828s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 851.331327] env[62499]: DEBUG oslo_concurrency.lockutils [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 20.250s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 851.331619] env[62499]: DEBUG nova.objects.instance [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lazy-loading 'resources' on Instance uuid 39bc8b3a-e9d9-48e3-a193-5fbdc2454346 {{(pid=62499) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 851.355033] env[62499]: INFO nova.scheduler.client.report [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Deleted allocations for instance 57d21761-431b-4a41-86eb-038f8c35d8ef [ 851.520224] env[62499]: DEBUG nova.compute.manager [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 851.545738] env[62499]: DEBUG nova.virt.hardware [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 851.545981] env[62499]: DEBUG nova.virt.hardware [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 851.546155] env[62499]: DEBUG nova.virt.hardware [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 851.546338] env[62499]: DEBUG nova.virt.hardware [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 851.546483] env[62499]: DEBUG nova.virt.hardware [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 851.546631] env[62499]: DEBUG nova.virt.hardware [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 851.546833] env[62499]: DEBUG nova.virt.hardware [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 851.546995] env[62499]: DEBUG nova.virt.hardware [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 851.547177] env[62499]: DEBUG nova.virt.hardware [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 851.547340] env[62499]: DEBUG nova.virt.hardware [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 851.547515] env[62499]: DEBUG nova.virt.hardware [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 851.548689] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5209c2d9-f075-4654-9a1d-30050992359a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 851.552358] env[62499]: DEBUG nova.network.neutron [req-97b84cb3-5a7f-4ed4-bf76-14d90bdfbe80 req-013ba8f0-626e-4741-b18c-99e372baf6f1 service nova] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 851.558549] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d0ed62e-0993-4f7a-8c0c-eec5e3afe97b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 851.571801] env[62499]: ERROR nova.compute.manager [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port dee6d343-e56a-4d3a-9f5f-2a6fe259825e, please check neutron logs for more information. [ 851.571801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Traceback (most recent call last): [ 851.571801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 851.571801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] yield resources [ 851.571801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 851.571801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] self.driver.spawn(context, instance, image_meta, [ 851.571801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 851.571801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] self._vmops.spawn(context, instance, image_meta, injected_files, [ 851.571801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 851.571801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] vm_ref = self.build_virtual_machine(instance, [ 851.571801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 851.572248] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] vif_infos = vmwarevif.get_vif_info(self._session, [ 851.572248] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 851.572248] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] for vif in network_info: [ 851.572248] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 851.572248] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] return self._sync_wrapper(fn, *args, **kwargs) [ 851.572248] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 851.572248] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] self.wait() [ 851.572248] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 851.572248] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] self[:] = self._gt.wait() [ 851.572248] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 851.572248] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] return self._exit_event.wait() [ 851.572248] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 851.572248] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] current.throw(*self._exc) [ 851.572609] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 851.572609] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] result = function(*args, **kwargs) [ 851.572609] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 851.572609] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] return func(*args, **kwargs) [ 851.572609] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 851.572609] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] raise e [ 851.572609] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 851.572609] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] nwinfo = self.network_api.allocate_for_instance( [ 851.572609] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 851.572609] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] created_port_ids = self._update_ports_for_instance( [ 851.572609] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 851.572609] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] with excutils.save_and_reraise_exception(): [ 851.572609] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 851.572977] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] self.force_reraise() [ 851.572977] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 851.572977] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] raise self.value [ 851.572977] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 851.572977] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] updated_port = self._update_port( [ 851.572977] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 851.572977] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] _ensure_no_port_binding_failure(port) [ 851.572977] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 851.572977] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] raise exception.PortBindingFailed(port_id=port['id']) [ 851.572977] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] nova.exception.PortBindingFailed: Binding failed for port dee6d343-e56a-4d3a-9f5f-2a6fe259825e, please check neutron logs for more information. [ 851.572977] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] [ 851.572977] env[62499]: INFO nova.compute.manager [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Terminating instance [ 851.574080] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Acquiring lock "refresh_cache-4eda80db-5db7-4fd6-9932-108c22f4a616" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 851.619032] env[62499]: DEBUG nova.network.neutron [req-97b84cb3-5a7f-4ed4-bf76-14d90bdfbe80 req-013ba8f0-626e-4741-b18c-99e372baf6f1 service nova] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 851.865643] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e9aa9e28-3f69-49df-b4ed-0665d7af58fe tempest-ServersAaction247Test-801381229 tempest-ServersAaction247Test-801381229-project-member] Lock "57d21761-431b-4a41-86eb-038f8c35d8ef" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 26.008s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 852.066050] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-422ad84c-1033-48b9-b24b-1a126f028398 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 852.072966] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93798d1b-4ca4-4079-9c79-12468fbc07fb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 852.101990] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f6ca154-7c09-4487-a27b-ccf6d8e046c9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 852.109412] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9b617fb8-8f4f-46d6-9a31-409c08a19506 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 852.122116] env[62499]: DEBUG oslo_concurrency.lockutils [req-97b84cb3-5a7f-4ed4-bf76-14d90bdfbe80 req-013ba8f0-626e-4741-b18c-99e372baf6f1 service nova] Releasing lock "refresh_cache-4eda80db-5db7-4fd6-9932-108c22f4a616" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 852.122692] env[62499]: DEBUG nova.compute.provider_tree [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 852.123972] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Acquired lock "refresh_cache-4eda80db-5db7-4fd6-9932-108c22f4a616" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 852.124175] env[62499]: DEBUG nova.network.neutron [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 852.627046] env[62499]: DEBUG nova.scheduler.client.report [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 852.648266] env[62499]: DEBUG nova.network.neutron [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 852.690682] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 852.691010] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 852.735692] env[62499]: DEBUG nova.network.neutron [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 853.052317] env[62499]: DEBUG nova.compute.manager [req-c6b65f4b-bbd7-45a3-8e08-7b26cf1a8f2e req-ceb22efb-4a0c-4522-9e9b-590b00476509 service nova] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Received event network-vif-deleted-dee6d343-e56a-4d3a-9f5f-2a6fe259825e {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 853.132190] env[62499]: DEBUG oslo_concurrency.lockutils [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.802s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 853.135032] env[62499]: DEBUG oslo_concurrency.lockutils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 21.742s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 853.155509] env[62499]: INFO nova.scheduler.client.report [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Deleted allocations for instance 39bc8b3a-e9d9-48e3-a193-5fbdc2454346 [ 853.196894] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 853.197464] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Starting heal instance info cache {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9928}} [ 853.197464] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Rebuilding the list of instances to heal {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 853.237794] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Releasing lock "refresh_cache-4eda80db-5db7-4fd6-9932-108c22f4a616" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 853.238268] env[62499]: DEBUG nova.compute.manager [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 853.238472] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 853.238762] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-153b6352-896f-41c3-ae64-ea12d961638f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 853.247869] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7effd6c5-b644-4c25-b889-72dd5e016af2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 853.269887] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4eda80db-5db7-4fd6-9932-108c22f4a616 could not be found. [ 853.270134] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 853.270315] env[62499]: INFO nova.compute.manager [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Took 0.03 seconds to destroy the instance on the hypervisor. [ 853.270558] env[62499]: DEBUG oslo.service.loopingcall [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 853.270783] env[62499]: DEBUG nova.compute.manager [-] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 853.270877] env[62499]: DEBUG nova.network.neutron [-] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 853.284237] env[62499]: DEBUG nova.network.neutron [-] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 853.662458] env[62499]: DEBUG oslo_concurrency.lockutils [None req-69219055-6f5d-4605-a5b4-2895fc048817 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "39bc8b3a-e9d9-48e3-a193-5fbdc2454346" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 26.304s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 853.701835] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 853.701987] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 853.702990] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: ad749135-08c1-48ca-af56-300d6e796012] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 853.702990] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 853.702990] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 853.702990] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 853.702990] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 853.702990] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Skipping network cache update for instance because it is Building. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9941}} [ 853.721867] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Acquiring lock "refresh_cache-008b230e-72b8-43ae-826a-b38111c56e76" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 853.722030] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Acquired lock "refresh_cache-008b230e-72b8-43ae-826a-b38111c56e76" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 853.722179] env[62499]: DEBUG nova.network.neutron [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Forcefully refreshing network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2061}} [ 853.722327] env[62499]: DEBUG nova.objects.instance [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lazy-loading 'info_cache' on Instance uuid 008b230e-72b8-43ae-826a-b38111c56e76 {{(pid=62499) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 853.787977] env[62499]: DEBUG nova.network.neutron [-] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 853.852095] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c7832fc-4566-4d39-893f-5a3409df15db {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 853.859497] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a0d9fb7-3317-4348-a92c-de43514a6428 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 853.890399] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a754bb79-9b20-43f7-8c5d-04fab96aa381 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 853.897466] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7d4363c-26a7-424c-b75a-dccb6380b54b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 853.910611] env[62499]: DEBUG nova.compute.provider_tree [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 854.140030] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "008b230e-72b8-43ae-826a-b38111c56e76" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 854.140030] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "008b230e-72b8-43ae-826a-b38111c56e76" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 854.140030] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "008b230e-72b8-43ae-826a-b38111c56e76-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 854.140030] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "008b230e-72b8-43ae-826a-b38111c56e76-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 854.140407] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "008b230e-72b8-43ae-826a-b38111c56e76-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 854.142187] env[62499]: INFO nova.compute.manager [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Terminating instance [ 854.143714] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "refresh_cache-008b230e-72b8-43ae-826a-b38111c56e76" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 854.290756] env[62499]: INFO nova.compute.manager [-] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Took 1.02 seconds to deallocate network for instance. [ 854.293245] env[62499]: DEBUG nova.compute.claims [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 854.293471] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 854.413833] env[62499]: DEBUG nova.scheduler.client.report [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 854.745388] env[62499]: DEBUG nova.network.neutron [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 854.918297] env[62499]: DEBUG oslo_concurrency.lockutils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.784s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 854.918952] env[62499]: ERROR nova.compute.manager [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 80575751-bae4-48a6-b43b-1a4c37d4b3fd, please check neutron logs for more information. [ 854.918952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Traceback (most recent call last): [ 854.918952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 854.918952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] self.driver.spawn(context, instance, image_meta, [ 854.918952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 854.918952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 854.918952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 854.918952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] vm_ref = self.build_virtual_machine(instance, [ 854.918952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 854.918952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] vif_infos = vmwarevif.get_vif_info(self._session, [ 854.918952] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 854.919339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] for vif in network_info: [ 854.919339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 854.919339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] return self._sync_wrapper(fn, *args, **kwargs) [ 854.919339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 854.919339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] self.wait() [ 854.919339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 854.919339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] self[:] = self._gt.wait() [ 854.919339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 854.919339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] return self._exit_event.wait() [ 854.919339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 854.919339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] current.throw(*self._exc) [ 854.919339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 854.919339] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] result = function(*args, **kwargs) [ 854.919752] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 854.919752] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] return func(*args, **kwargs) [ 854.919752] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 854.919752] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] raise e [ 854.919752] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 854.919752] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] nwinfo = self.network_api.allocate_for_instance( [ 854.919752] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 854.919752] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] created_port_ids = self._update_ports_for_instance( [ 854.919752] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 854.919752] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] with excutils.save_and_reraise_exception(): [ 854.919752] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 854.919752] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] self.force_reraise() [ 854.919752] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 854.920169] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] raise self.value [ 854.920169] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 854.920169] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] updated_port = self._update_port( [ 854.920169] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 854.920169] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] _ensure_no_port_binding_failure(port) [ 854.920169] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 854.920169] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] raise exception.PortBindingFailed(port_id=port['id']) [ 854.920169] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] nova.exception.PortBindingFailed: Binding failed for port 80575751-bae4-48a6-b43b-1a4c37d4b3fd, please check neutron logs for more information. [ 854.920169] env[62499]: ERROR nova.compute.manager [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] [ 854.920169] env[62499]: DEBUG nova.compute.utils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Binding failed for port 80575751-bae4-48a6-b43b-1a4c37d4b3fd, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 854.920848] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 22.268s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 854.921109] env[62499]: DEBUG nova.objects.instance [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Lazy-loading 'resources' on Instance uuid 15f8e04f-8c14-4315-8686-da4db517f7b6 {{(pid=62499) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 854.922307] env[62499]: DEBUG nova.compute.manager [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Build of instance 49f9108c-7256-4a2c-9ffd-a6d041a180e4 was re-scheduled: Binding failed for port 80575751-bae4-48a6-b43b-1a4c37d4b3fd, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 854.922710] env[62499]: DEBUG nova.compute.manager [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 854.922924] env[62499]: DEBUG oslo_concurrency.lockutils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Acquiring lock "refresh_cache-49f9108c-7256-4a2c-9ffd-a6d041a180e4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 854.923120] env[62499]: DEBUG oslo_concurrency.lockutils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Acquired lock "refresh_cache-49f9108c-7256-4a2c-9ffd-a6d041a180e4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 854.923285] env[62499]: DEBUG nova.network.neutron [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 855.326243] env[62499]: DEBUG nova.network.neutron [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 855.441548] env[62499]: DEBUG nova.network.neutron [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 855.485187] env[62499]: DEBUG nova.network.neutron [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 855.647535] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-17677366-28a9-4976-b29d-f4a688ea135e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 855.654955] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74d61a1c-1532-44ad-b5f3-bfec2238c6d3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 855.684418] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-57b573a3-360e-49d9-b882-13eb5b023954 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 855.690934] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4ba3869-d87a-443a-9c44-5fcd308a9f3a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 855.703245] env[62499]: DEBUG nova.compute.provider_tree [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 855.829062] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Releasing lock "refresh_cache-008b230e-72b8-43ae-826a-b38111c56e76" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 855.829062] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Updated the network info_cache for instance {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9999}} [ 855.829297] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquired lock "refresh_cache-008b230e-72b8-43ae-826a-b38111c56e76" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 855.829468] env[62499]: DEBUG nova.network.neutron [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 855.830611] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 855.832532] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 855.832532] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 855.832532] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 855.832532] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 855.832532] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 855.832532] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62499) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10547}} [ 855.832778] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager.update_available_resource {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 855.989029] env[62499]: DEBUG oslo_concurrency.lockutils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Releasing lock "refresh_cache-49f9108c-7256-4a2c-9ffd-a6d041a180e4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 855.989029] env[62499]: DEBUG nova.compute.manager [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 855.989029] env[62499]: DEBUG nova.compute.manager [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 855.989029] env[62499]: DEBUG nova.network.neutron [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 856.005736] env[62499]: DEBUG nova.network.neutron [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 856.205996] env[62499]: DEBUG nova.scheduler.client.report [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 856.334529] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 856.350789] env[62499]: DEBUG nova.network.neutron [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 856.405970] env[62499]: DEBUG nova.network.neutron [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 856.508061] env[62499]: DEBUG nova.network.neutron [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 856.711242] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.790s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 856.713504] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 23.578s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 856.727540] env[62499]: INFO nova.scheduler.client.report [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Deleted allocations for instance 15f8e04f-8c14-4315-8686-da4db517f7b6 [ 856.908903] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Releasing lock "refresh_cache-008b230e-72b8-43ae-826a-b38111c56e76" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 856.909350] env[62499]: DEBUG nova.compute.manager [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 856.909543] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 856.910511] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd9bbbf1-fc77-426d-b894-3d3866df498c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 856.918553] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Powering off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 856.918797] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-6b011bdd-303e-4193-9106-046e7943f1b5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 856.925865] env[62499]: DEBUG oslo_vmware.api [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 856.925865] env[62499]: value = "task-1335766" [ 856.925865] env[62499]: _type = "Task" [ 856.925865] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 856.937110] env[62499]: DEBUG oslo_vmware.api [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335766, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 857.011159] env[62499]: INFO nova.compute.manager [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] [instance: 49f9108c-7256-4a2c-9ffd-a6d041a180e4] Took 1.02 seconds to deallocate network for instance. [ 857.234458] env[62499]: DEBUG oslo_concurrency.lockutils [None req-9460dbb2-d889-4d00-a7c9-6c3654c6287a tempest-ServerShowV254Test-642463904 tempest-ServerShowV254Test-642463904-project-member] Lock "15f8e04f-8c14-4315-8686-da4db517f7b6" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 28.295s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 857.427549] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b9d4f3e-c423-41ba-9865-c6da73b73851 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.440086] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3e5960e-faa5-4714-a5d5-dcc962c4dbe3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.442919] env[62499]: DEBUG oslo_vmware.api [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335766, 'name': PowerOffVM_Task, 'duration_secs': 0.11328} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 857.443217] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Powered off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 857.443342] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Unregistering the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 857.443867] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-825e0983-28f5-4791-bad5-c14fab148ea0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.472556] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f265712-5565-470e-a6e6-ca825d788ba9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.475107] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Unregistered the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 857.475304] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Deleting contents of the VM from datastore datastore2 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 857.475475] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Deleting the datastore file [datastore2] 008b230e-72b8-43ae-826a-b38111c56e76 {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 857.475701] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-b86f4b37-593d-468b-8403-8506764192d2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.483168] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fbb2e592-62b5-42c7-aef2-cea81fb41f12 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.486953] env[62499]: DEBUG oslo_vmware.api [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for the task: (returnval){ [ 857.486953] env[62499]: value = "task-1335768" [ 857.486953] env[62499]: _type = "Task" [ 857.486953] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 857.499647] env[62499]: DEBUG nova.compute.provider_tree [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 857.504760] env[62499]: DEBUG oslo_vmware.api [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335768, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 857.997175] env[62499]: DEBUG oslo_vmware.api [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Task: {'id': task-1335768, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.088974} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 857.997436] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Deleted the datastore file {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 857.997620] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Deleted contents of the VM from datastore datastore2 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 857.997797] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 857.998007] env[62499]: INFO nova.compute.manager [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Took 1.09 seconds to destroy the instance on the hypervisor. [ 857.998261] env[62499]: DEBUG oslo.service.loopingcall [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 857.998457] env[62499]: DEBUG nova.compute.manager [-] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 857.998563] env[62499]: DEBUG nova.network.neutron [-] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 858.002301] env[62499]: DEBUG nova.scheduler.client.report [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 858.020842] env[62499]: DEBUG nova.network.neutron [-] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 858.043921] env[62499]: INFO nova.scheduler.client.report [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Deleted allocations for instance 49f9108c-7256-4a2c-9ffd-a6d041a180e4 [ 858.508052] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.794s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 858.508052] env[62499]: ERROR nova.compute.manager [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b80023b9-103c-4a15-948f-4139319d4feb, please check neutron logs for more information. [ 858.508052] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Traceback (most recent call last): [ 858.508052] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 858.508052] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] self.driver.spawn(context, instance, image_meta, [ 858.508052] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 858.508052] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 858.508052] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 858.508052] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] vm_ref = self.build_virtual_machine(instance, [ 858.508586] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 858.508586] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] vif_infos = vmwarevif.get_vif_info(self._session, [ 858.508586] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 858.508586] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] for vif in network_info: [ 858.508586] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 858.508586] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] return self._sync_wrapper(fn, *args, **kwargs) [ 858.508586] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 858.508586] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] self.wait() [ 858.508586] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 858.508586] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] self[:] = self._gt.wait() [ 858.508586] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 858.508586] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] return self._exit_event.wait() [ 858.508586] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 858.508953] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] current.throw(*self._exc) [ 858.508953] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 858.508953] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] result = function(*args, **kwargs) [ 858.508953] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 858.508953] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] return func(*args, **kwargs) [ 858.508953] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 858.508953] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] raise e [ 858.508953] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 858.508953] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] nwinfo = self.network_api.allocate_for_instance( [ 858.508953] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 858.508953] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] created_port_ids = self._update_ports_for_instance( [ 858.508953] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 858.508953] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] with excutils.save_and_reraise_exception(): [ 858.509319] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 858.509319] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] self.force_reraise() [ 858.509319] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 858.509319] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] raise self.value [ 858.509319] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 858.509319] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] updated_port = self._update_port( [ 858.509319] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 858.509319] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] _ensure_no_port_binding_failure(port) [ 858.509319] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 858.509319] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] raise exception.PortBindingFailed(port_id=port['id']) [ 858.509319] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] nova.exception.PortBindingFailed: Binding failed for port b80023b9-103c-4a15-948f-4139319d4feb, please check neutron logs for more information. [ 858.509319] env[62499]: ERROR nova.compute.manager [instance: d650a1a5-3706-4682-a813-f85ea23098e2] [ 858.509619] env[62499]: DEBUG nova.compute.utils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Binding failed for port b80023b9-103c-4a15-948f-4139319d4feb, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 858.510264] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 22.889s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 858.517028] env[62499]: DEBUG nova.compute.manager [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Build of instance d650a1a5-3706-4682-a813-f85ea23098e2 was re-scheduled: Binding failed for port b80023b9-103c-4a15-948f-4139319d4feb, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 858.517028] env[62499]: DEBUG nova.compute.manager [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 858.517308] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Acquiring lock "refresh_cache-d650a1a5-3706-4682-a813-f85ea23098e2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 858.517514] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Acquired lock "refresh_cache-d650a1a5-3706-4682-a813-f85ea23098e2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 858.517680] env[62499]: DEBUG nova.network.neutron [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 858.523067] env[62499]: DEBUG nova.network.neutron [-] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 858.552144] env[62499]: DEBUG oslo_concurrency.lockutils [None req-bf7ce378-1738-4f88-95f8-a43cfe634856 tempest-ListImageFiltersTestJSON-1745683704 tempest-ListImageFiltersTestJSON-1745683704-project-member] Lock "49f9108c-7256-4a2c-9ffd-a6d041a180e4" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 157.355s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 859.025798] env[62499]: INFO nova.compute.manager [-] [instance: 008b230e-72b8-43ae-826a-b38111c56e76] Took 1.03 seconds to deallocate network for instance. [ 859.039636] env[62499]: DEBUG nova.network.neutron [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 859.056358] env[62499]: DEBUG nova.compute.manager [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 859.063078] env[62499]: DEBUG oslo_concurrency.lockutils [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquiring lock "b534fcdb-b0db-4e07-85fc-696dc58168aa" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 859.063315] env[62499]: DEBUG oslo_concurrency.lockutils [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Lock "b534fcdb-b0db-4e07-85fc-696dc58168aa" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 859.131760] env[62499]: DEBUG nova.network.neutron [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 859.247032] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4bfb8a1d-2691-429a-bfb4-75bc1f6d18ad {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.254443] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c08826e6-f891-4d1c-a647-58ca6838b0b5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.287971] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d7510b8-8f42-4dcd-9bfd-b6c613ae2d07 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.295158] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33c56b7c-c0f7-4ee3-bec9-e37b57f90107 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.307864] env[62499]: DEBUG nova.compute.provider_tree [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 859.533704] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 859.584053] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 859.638034] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Releasing lock "refresh_cache-d650a1a5-3706-4682-a813-f85ea23098e2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 859.638034] env[62499]: DEBUG nova.compute.manager [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 859.638034] env[62499]: DEBUG nova.compute.manager [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 859.638034] env[62499]: DEBUG nova.network.neutron [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 859.724971] env[62499]: DEBUG nova.network.neutron [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 859.813096] env[62499]: DEBUG nova.scheduler.client.report [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 860.231192] env[62499]: DEBUG nova.network.neutron [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 860.318600] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.809s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 860.319332] env[62499]: ERROR nova.compute.manager [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d8305010-f9eb-4feb-8f48-828f581c7b97, please check neutron logs for more information. [ 860.319332] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Traceback (most recent call last): [ 860.319332] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 860.319332] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] self.driver.spawn(context, instance, image_meta, [ 860.319332] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 860.319332] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] self._vmops.spawn(context, instance, image_meta, injected_files, [ 860.319332] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 860.319332] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] vm_ref = self.build_virtual_machine(instance, [ 860.319332] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 860.319332] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] vif_infos = vmwarevif.get_vif_info(self._session, [ 860.319332] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 860.319732] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] for vif in network_info: [ 860.319732] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 860.319732] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] return self._sync_wrapper(fn, *args, **kwargs) [ 860.319732] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 860.319732] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] self.wait() [ 860.319732] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 860.319732] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] self[:] = self._gt.wait() [ 860.319732] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 860.319732] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] return self._exit_event.wait() [ 860.319732] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 860.319732] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] result = hub.switch() [ 860.319732] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 860.319732] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] return self.greenlet.switch() [ 860.320161] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 860.320161] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] result = function(*args, **kwargs) [ 860.320161] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 860.320161] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] return func(*args, **kwargs) [ 860.320161] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 860.320161] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] raise e [ 860.320161] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 860.320161] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] nwinfo = self.network_api.allocate_for_instance( [ 860.320161] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 860.320161] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] created_port_ids = self._update_ports_for_instance( [ 860.320161] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 860.320161] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] with excutils.save_and_reraise_exception(): [ 860.320161] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 860.320540] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] self.force_reraise() [ 860.320540] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 860.320540] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] raise self.value [ 860.320540] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 860.320540] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] updated_port = self._update_port( [ 860.320540] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 860.320540] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] _ensure_no_port_binding_failure(port) [ 860.320540] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 860.320540] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] raise exception.PortBindingFailed(port_id=port['id']) [ 860.320540] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] nova.exception.PortBindingFailed: Binding failed for port d8305010-f9eb-4feb-8f48-828f581c7b97, please check neutron logs for more information. [ 860.320540] env[62499]: ERROR nova.compute.manager [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] [ 860.320883] env[62499]: DEBUG nova.compute.utils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Binding failed for port d8305010-f9eb-4feb-8f48-828f581c7b97, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 860.321472] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 22.243s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 860.324573] env[62499]: DEBUG nova.compute.manager [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Build of instance 60a4b8c3-9dc4-4867-adca-4503e61ce237 was re-scheduled: Binding failed for port d8305010-f9eb-4feb-8f48-828f581c7b97, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 860.325187] env[62499]: DEBUG nova.compute.manager [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 860.325459] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Acquiring lock "refresh_cache-60a4b8c3-9dc4-4867-adca-4503e61ce237" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 860.325618] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Acquired lock "refresh_cache-60a4b8c3-9dc4-4867-adca-4503e61ce237" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 860.325939] env[62499]: DEBUG nova.network.neutron [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 860.733987] env[62499]: INFO nova.compute.manager [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] [instance: d650a1a5-3706-4682-a813-f85ea23098e2] Took 1.10 seconds to deallocate network for instance. [ 860.862921] env[62499]: DEBUG nova.network.neutron [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 860.964343] env[62499]: DEBUG nova.network.neutron [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 861.092952] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d19df999-afb9-44d5-869f-d2386c5315d3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 861.101238] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-425ef2e3-7783-4002-8232-735d3761e920 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 861.133029] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f4372b7a-1543-4c40-a5d5-8f1704ca1075 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 861.140759] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-65c262d9-c0a8-4599-b66b-28b53a013d5f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 861.153989] env[62499]: DEBUG nova.compute.provider_tree [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 861.466931] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Releasing lock "refresh_cache-60a4b8c3-9dc4-4867-adca-4503e61ce237" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 861.467203] env[62499]: DEBUG nova.compute.manager [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 861.467390] env[62499]: DEBUG nova.compute.manager [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 861.467564] env[62499]: DEBUG nova.network.neutron [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 861.482754] env[62499]: DEBUG nova.network.neutron [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 861.656790] env[62499]: DEBUG nova.scheduler.client.report [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 861.763846] env[62499]: INFO nova.scheduler.client.report [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Deleted allocations for instance d650a1a5-3706-4682-a813-f85ea23098e2 [ 861.985660] env[62499]: DEBUG nova.network.neutron [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 862.161953] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.840s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 862.162634] env[62499]: ERROR nova.compute.manager [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 69123df3-56d1-490c-9303-b6427e432f7f, please check neutron logs for more information. [ 862.162634] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] Traceback (most recent call last): [ 862.162634] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 862.162634] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] self.driver.spawn(context, instance, image_meta, [ 862.162634] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 862.162634] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] self._vmops.spawn(context, instance, image_meta, injected_files, [ 862.162634] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 862.162634] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] vm_ref = self.build_virtual_machine(instance, [ 862.162634] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 862.162634] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] vif_infos = vmwarevif.get_vif_info(self._session, [ 862.162634] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 862.162967] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] for vif in network_info: [ 862.162967] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 862.162967] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] return self._sync_wrapper(fn, *args, **kwargs) [ 862.162967] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 862.162967] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] self.wait() [ 862.162967] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 862.162967] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] self[:] = self._gt.wait() [ 862.162967] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 862.162967] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] return self._exit_event.wait() [ 862.162967] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 862.162967] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] current.throw(*self._exc) [ 862.162967] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 862.162967] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] result = function(*args, **kwargs) [ 862.163296] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 862.163296] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] return func(*args, **kwargs) [ 862.163296] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 862.163296] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] raise e [ 862.163296] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 862.163296] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] nwinfo = self.network_api.allocate_for_instance( [ 862.163296] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 862.163296] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] created_port_ids = self._update_ports_for_instance( [ 862.163296] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 862.163296] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] with excutils.save_and_reraise_exception(): [ 862.163296] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 862.163296] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] self.force_reraise() [ 862.163296] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 862.163605] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] raise self.value [ 862.163605] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 862.163605] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] updated_port = self._update_port( [ 862.163605] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 862.163605] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] _ensure_no_port_binding_failure(port) [ 862.163605] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 862.163605] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] raise exception.PortBindingFailed(port_id=port['id']) [ 862.163605] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] nova.exception.PortBindingFailed: Binding failed for port 69123df3-56d1-490c-9303-b6427e432f7f, please check neutron logs for more information. [ 862.163605] env[62499]: ERROR nova.compute.manager [instance: ad749135-08c1-48ca-af56-300d6e796012] [ 862.163605] env[62499]: DEBUG nova.compute.utils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Binding failed for port 69123df3-56d1-490c-9303-b6427e432f7f, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 862.167735] env[62499]: DEBUG oslo_concurrency.lockutils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 21.705s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 862.180209] env[62499]: DEBUG nova.compute.manager [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Build of instance ad749135-08c1-48ca-af56-300d6e796012 was re-scheduled: Binding failed for port 69123df3-56d1-490c-9303-b6427e432f7f, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 862.180209] env[62499]: DEBUG nova.compute.manager [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 862.180209] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Acquiring lock "refresh_cache-ad749135-08c1-48ca-af56-300d6e796012" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 862.180209] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Acquired lock "refresh_cache-ad749135-08c1-48ca-af56-300d6e796012" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 862.180461] env[62499]: DEBUG nova.network.neutron [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 862.273706] env[62499]: DEBUG oslo_concurrency.lockutils [None req-e48a4632-10ee-406f-b432-282041b25d20 tempest-ServerRescueTestJSONUnderV235-1097481186 tempest-ServerRescueTestJSONUnderV235-1097481186-project-member] Lock "d650a1a5-3706-4682-a813-f85ea23098e2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 159.484s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 862.492885] env[62499]: INFO nova.compute.manager [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: 60a4b8c3-9dc4-4867-adca-4503e61ce237] Took 1.02 seconds to deallocate network for instance. [ 862.718853] env[62499]: DEBUG nova.network.neutron [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 862.777616] env[62499]: DEBUG nova.compute.manager [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 862.838949] env[62499]: DEBUG nova.network.neutron [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 863.036101] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-257bddd8-d583-4ade-87f8-d719730cb0df {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 863.042776] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60a5b327-0ba6-4473-b64b-4124963b2e42 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 863.074074] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1fe0258e-71ea-4651-89a2-df0985a80ebb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 863.081758] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3fdb91ee-eeac-4df6-9f22-bdd055bf5ff5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 863.096169] env[62499]: DEBUG nova.compute.provider_tree [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 863.301099] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 863.341929] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Releasing lock "refresh_cache-ad749135-08c1-48ca-af56-300d6e796012" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 863.342208] env[62499]: DEBUG nova.compute.manager [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 863.342424] env[62499]: DEBUG nova.compute.manager [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 863.342601] env[62499]: DEBUG nova.network.neutron [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 863.358724] env[62499]: DEBUG nova.network.neutron [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 863.539168] env[62499]: INFO nova.scheduler.client.report [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Deleted allocations for instance 60a4b8c3-9dc4-4867-adca-4503e61ce237 [ 863.598739] env[62499]: DEBUG nova.scheduler.client.report [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 863.862620] env[62499]: DEBUG nova.network.neutron [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 864.048739] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1a70708b-ce1e-442f-8855-8a06d7436083 tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Lock "60a4b8c3-9dc4-4867-adca-4503e61ce237" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 151.293s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 864.103662] env[62499]: DEBUG oslo_concurrency.lockutils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.936s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 864.104289] env[62499]: ERROR nova.compute.manager [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b7ead464-8575-4153-96fc-fab89ca560f8, please check neutron logs for more information. [ 864.104289] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Traceback (most recent call last): [ 864.104289] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 864.104289] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] self.driver.spawn(context, instance, image_meta, [ 864.104289] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 864.104289] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 864.104289] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 864.104289] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] vm_ref = self.build_virtual_machine(instance, [ 864.104289] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 864.104289] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] vif_infos = vmwarevif.get_vif_info(self._session, [ 864.104289] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 864.104590] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] for vif in network_info: [ 864.104590] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 864.104590] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] return self._sync_wrapper(fn, *args, **kwargs) [ 864.104590] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 864.104590] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] self.wait() [ 864.104590] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 864.104590] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] self[:] = self._gt.wait() [ 864.104590] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 864.104590] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] return self._exit_event.wait() [ 864.104590] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 864.104590] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] current.throw(*self._exc) [ 864.104590] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 864.104590] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] result = function(*args, **kwargs) [ 864.104900] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 864.104900] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] return func(*args, **kwargs) [ 864.104900] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 864.104900] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] raise e [ 864.104900] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 864.104900] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] nwinfo = self.network_api.allocate_for_instance( [ 864.104900] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 864.104900] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] created_port_ids = self._update_ports_for_instance( [ 864.104900] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 864.104900] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] with excutils.save_and_reraise_exception(): [ 864.104900] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 864.104900] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] self.force_reraise() [ 864.104900] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 864.105215] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] raise self.value [ 864.105215] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 864.105215] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] updated_port = self._update_port( [ 864.105215] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 864.105215] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] _ensure_no_port_binding_failure(port) [ 864.105215] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 864.105215] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] raise exception.PortBindingFailed(port_id=port['id']) [ 864.105215] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] nova.exception.PortBindingFailed: Binding failed for port b7ead464-8575-4153-96fc-fab89ca560f8, please check neutron logs for more information. [ 864.105215] env[62499]: ERROR nova.compute.manager [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] [ 864.105215] env[62499]: DEBUG nova.compute.utils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Binding failed for port b7ead464-8575-4153-96fc-fab89ca560f8, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 864.106289] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 20.257s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 864.112452] env[62499]: DEBUG nova.compute.manager [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Build of instance 2a1c52ab-8ee6-4b63-ad04-2450a60262d0 was re-scheduled: Binding failed for port b7ead464-8575-4153-96fc-fab89ca560f8, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 864.112899] env[62499]: DEBUG nova.compute.manager [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 864.113147] env[62499]: DEBUG oslo_concurrency.lockutils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Acquiring lock "refresh_cache-2a1c52ab-8ee6-4b63-ad04-2450a60262d0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 864.114021] env[62499]: DEBUG oslo_concurrency.lockutils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Acquired lock "refresh_cache-2a1c52ab-8ee6-4b63-ad04-2450a60262d0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 864.114021] env[62499]: DEBUG nova.network.neutron [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 864.370121] env[62499]: INFO nova.compute.manager [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] [instance: ad749135-08c1-48ca-af56-300d6e796012] Took 1.03 seconds to deallocate network for instance. [ 864.551537] env[62499]: DEBUG nova.compute.manager [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 864.636755] env[62499]: DEBUG nova.scheduler.client.report [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Refreshing inventories for resource provider 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:818}} [ 864.639558] env[62499]: DEBUG nova.network.neutron [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 864.657692] env[62499]: DEBUG nova.scheduler.client.report [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Updating ProviderTree inventory for provider 38a6c941-9755-44af-8da8-6ef6281d0157 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:782}} [ 864.657953] env[62499]: DEBUG nova.compute.provider_tree [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Updating inventory in ProviderTree for provider 38a6c941-9755-44af-8da8-6ef6281d0157 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 864.671418] env[62499]: DEBUG nova.scheduler.client.report [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Refreshing aggregate associations for resource provider 38a6c941-9755-44af-8da8-6ef6281d0157, aggregates: None {{(pid=62499) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:827}} [ 864.697957] env[62499]: DEBUG nova.scheduler.client.report [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Refreshing trait associations for resource provider 38a6c941-9755-44af-8da8-6ef6281d0157, traits: COMPUTE_NET_ATTACH_INTERFACE,HW_ARCH_X86_64,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NODE {{(pid=62499) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:839}} [ 864.771438] env[62499]: DEBUG nova.network.neutron [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 864.987234] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-786328be-32c6-4888-af0e-187554491347 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.996843] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c995de8-496b-4c6c-8f3c-ab7015c8e1d5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 865.031367] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8a41d43-2387-4da5-8bfb-a6fe7b0a5c93 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 865.041129] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d6ea47c-2f76-413a-b336-027fc89b64e0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 865.052735] env[62499]: DEBUG nova.compute.provider_tree [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 865.070206] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 865.277688] env[62499]: DEBUG oslo_concurrency.lockutils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Releasing lock "refresh_cache-2a1c52ab-8ee6-4b63-ad04-2450a60262d0" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 865.278014] env[62499]: DEBUG nova.compute.manager [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 865.278209] env[62499]: DEBUG nova.compute.manager [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 865.278378] env[62499]: DEBUG nova.network.neutron [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 865.292929] env[62499]: DEBUG nova.network.neutron [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 865.404485] env[62499]: INFO nova.scheduler.client.report [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Deleted allocations for instance ad749135-08c1-48ca-af56-300d6e796012 [ 865.556489] env[62499]: DEBUG nova.scheduler.client.report [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 865.795910] env[62499]: DEBUG nova.network.neutron [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 865.817276] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Acquiring lock "f1b5512b-702f-45b1-9844-13a1218c31b3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 865.817513] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Lock "f1b5512b-702f-45b1-9844-13a1218c31b3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 865.915610] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c07b55e9-c29b-4c97-a3b8-b6f485ef2cce tempest-ServerRescueNegativeTestJSON-1957428111 tempest-ServerRescueNegativeTestJSON-1957428111-project-member] Lock "ad749135-08c1-48ca-af56-300d6e796012" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 152.710s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.061435] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.955s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.062508] env[62499]: ERROR nova.compute.manager [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 392f73a3-4972-4a0b-a13a-fef5efe59658, please check neutron logs for more information. [ 866.062508] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Traceback (most recent call last): [ 866.062508] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 866.062508] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] self.driver.spawn(context, instance, image_meta, [ 866.062508] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 866.062508] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] self._vmops.spawn(context, instance, image_meta, injected_files, [ 866.062508] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 866.062508] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] vm_ref = self.build_virtual_machine(instance, [ 866.062508] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 866.062508] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] vif_infos = vmwarevif.get_vif_info(self._session, [ 866.062508] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 866.062881] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] for vif in network_info: [ 866.062881] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 866.062881] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] return self._sync_wrapper(fn, *args, **kwargs) [ 866.062881] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 866.062881] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] self.wait() [ 866.062881] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 866.062881] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] self[:] = self._gt.wait() [ 866.062881] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 866.062881] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] return self._exit_event.wait() [ 866.062881] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 866.062881] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] current.throw(*self._exc) [ 866.062881] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 866.062881] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] result = function(*args, **kwargs) [ 866.063258] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 866.063258] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] return func(*args, **kwargs) [ 866.063258] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 866.063258] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] raise e [ 866.063258] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 866.063258] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] nwinfo = self.network_api.allocate_for_instance( [ 866.063258] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 866.063258] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] created_port_ids = self._update_ports_for_instance( [ 866.063258] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 866.063258] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] with excutils.save_and_reraise_exception(): [ 866.063258] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 866.063258] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] self.force_reraise() [ 866.063258] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 866.063630] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] raise self.value [ 866.063630] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 866.063630] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] updated_port = self._update_port( [ 866.063630] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 866.063630] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] _ensure_no_port_binding_failure(port) [ 866.063630] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 866.063630] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] raise exception.PortBindingFailed(port_id=port['id']) [ 866.063630] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] nova.exception.PortBindingFailed: Binding failed for port 392f73a3-4972-4a0b-a13a-fef5efe59658, please check neutron logs for more information. [ 866.063630] env[62499]: ERROR nova.compute.manager [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] [ 866.063630] env[62499]: DEBUG nova.compute.utils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Binding failed for port 392f73a3-4972-4a0b-a13a-fef5efe59658, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 866.064970] env[62499]: DEBUG nova.compute.manager [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Build of instance 6da95995-0527-4b05-94dc-a4fcf0ae2f92 was re-scheduled: Binding failed for port 392f73a3-4972-4a0b-a13a-fef5efe59658, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 866.065393] env[62499]: DEBUG nova.compute.manager [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 866.065619] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquiring lock "refresh_cache-6da95995-0527-4b05-94dc-a4fcf0ae2f92" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 866.065772] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Acquired lock "refresh_cache-6da95995-0527-4b05-94dc-a4fcf0ae2f92" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 866.065932] env[62499]: DEBUG nova.network.neutron [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 866.067277] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 20.995s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.068836] env[62499]: INFO nova.compute.claims [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 866.299822] env[62499]: INFO nova.compute.manager [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] [instance: 2a1c52ab-8ee6-4b63-ad04-2450a60262d0] Took 1.02 seconds to deallocate network for instance. [ 866.418538] env[62499]: DEBUG nova.compute.manager [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 866.591063] env[62499]: DEBUG nova.network.neutron [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 866.668786] env[62499]: DEBUG nova.network.neutron [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 866.942793] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 867.173057] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Releasing lock "refresh_cache-6da95995-0527-4b05-94dc-a4fcf0ae2f92" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 867.173303] env[62499]: DEBUG nova.compute.manager [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 867.174025] env[62499]: DEBUG nova.compute.manager [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 867.174025] env[62499]: DEBUG nova.network.neutron [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 867.203700] env[62499]: DEBUG nova.network.neutron [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 867.291360] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ff6608a-e814-4f3a-815e-8e585e253901 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 867.296848] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d5667e8-f917-4610-be94-233c751998e4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 867.328167] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7bec5702-9656-486a-936c-93197e066e28 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 867.339276] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd56d00a-97ee-42ec-a6c3-993eda31f2ff {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 867.346558] env[62499]: INFO nova.scheduler.client.report [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Deleted allocations for instance 2a1c52ab-8ee6-4b63-ad04-2450a60262d0 [ 867.367426] env[62499]: DEBUG nova.compute.provider_tree [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 867.709040] env[62499]: DEBUG nova.network.neutron [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 867.870148] env[62499]: DEBUG oslo_concurrency.lockutils [None req-708aeb0e-76d9-428b-96c9-50a39a2ef990 tempest-ServersTestFqdnHostnames-1879826867 tempest-ServersTestFqdnHostnames-1879826867-project-member] Lock "2a1c52ab-8ee6-4b63-ad04-2450a60262d0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 145.618s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 867.870983] env[62499]: DEBUG nova.scheduler.client.report [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 868.213251] env[62499]: INFO nova.compute.manager [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] [instance: 6da95995-0527-4b05-94dc-a4fcf0ae2f92] Took 1.04 seconds to deallocate network for instance. [ 868.382017] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.312s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 868.382017] env[62499]: DEBUG nova.compute.manager [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 868.386351] env[62499]: DEBUG oslo_concurrency.lockutils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 20.721s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 868.392508] env[62499]: DEBUG nova.compute.manager [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 868.898370] env[62499]: DEBUG nova.compute.utils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 868.904815] env[62499]: DEBUG nova.compute.manager [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 868.904953] env[62499]: DEBUG nova.network.neutron [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 868.921957] env[62499]: DEBUG oslo_concurrency.lockutils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 868.979705] env[62499]: DEBUG nova.policy [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd67d200795f843cc81b10074a30aae4a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9056b5a2acfa45a48b5b1530dd9b7860', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 869.142129] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1be6909d-8bfe-4879-b955-60874c206b1d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 869.149722] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-53922d98-46d3-4ce2-af71-d5c90a7164c8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 869.180654] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-366cb0ae-9e3b-49ce-b37e-9f87a6e6ce40 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 869.188594] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8435b2f-4077-4524-91c8-010dc0fe7165 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 869.202594] env[62499]: DEBUG nova.compute.provider_tree [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 869.242676] env[62499]: INFO nova.scheduler.client.report [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Deleted allocations for instance 6da95995-0527-4b05-94dc-a4fcf0ae2f92 [ 869.403715] env[62499]: DEBUG nova.compute.manager [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 869.487897] env[62499]: DEBUG nova.network.neutron [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Successfully created port: 296e37b0-c4ab-4591-9b5f-d51c80bb0f9c {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 869.707895] env[62499]: DEBUG nova.scheduler.client.report [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 869.755783] env[62499]: DEBUG oslo_concurrency.lockutils [None req-4641cb07-ea4d-4b99-9ae8-8b9b53720486 tempest-AttachInterfacesTestJSON-1875529461 tempest-AttachInterfacesTestJSON-1875529461-project-member] Lock "6da95995-0527-4b05-94dc-a4fcf0ae2f92" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 140.181s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 870.211752] env[62499]: DEBUG oslo_concurrency.lockutils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.829s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 870.212656] env[62499]: ERROR nova.compute.manager [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f0eb55e9-0018-4c0e-99cb-227551afe2c8, please check neutron logs for more information. [ 870.212656] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Traceback (most recent call last): [ 870.212656] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 870.212656] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] self.driver.spawn(context, instance, image_meta, [ 870.212656] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 870.212656] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 870.212656] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 870.212656] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] vm_ref = self.build_virtual_machine(instance, [ 870.212656] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 870.212656] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] vif_infos = vmwarevif.get_vif_info(self._session, [ 870.212656] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 870.213023] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] for vif in network_info: [ 870.213023] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 870.213023] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] return self._sync_wrapper(fn, *args, **kwargs) [ 870.213023] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 870.213023] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] self.wait() [ 870.213023] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 870.213023] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] self[:] = self._gt.wait() [ 870.213023] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 870.213023] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] return self._exit_event.wait() [ 870.213023] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 870.213023] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] current.throw(*self._exc) [ 870.213023] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 870.213023] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] result = function(*args, **kwargs) [ 870.213392] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 870.213392] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] return func(*args, **kwargs) [ 870.213392] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 870.213392] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] raise e [ 870.213392] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 870.213392] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] nwinfo = self.network_api.allocate_for_instance( [ 870.213392] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 870.213392] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] created_port_ids = self._update_ports_for_instance( [ 870.213392] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 870.213392] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] with excutils.save_and_reraise_exception(): [ 870.213392] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 870.213392] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] self.force_reraise() [ 870.213392] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 870.213823] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] raise self.value [ 870.213823] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 870.213823] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] updated_port = self._update_port( [ 870.213823] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 870.213823] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] _ensure_no_port_binding_failure(port) [ 870.213823] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 870.213823] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] raise exception.PortBindingFailed(port_id=port['id']) [ 870.213823] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] nova.exception.PortBindingFailed: Binding failed for port f0eb55e9-0018-4c0e-99cb-227551afe2c8, please check neutron logs for more information. [ 870.213823] env[62499]: ERROR nova.compute.manager [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] [ 870.214140] env[62499]: DEBUG nova.compute.utils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Binding failed for port f0eb55e9-0018-4c0e-99cb-227551afe2c8, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 870.217642] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 20.841s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 870.220650] env[62499]: DEBUG nova.compute.manager [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Build of instance 263931cd-b2dc-41bb-8a2b-abf61aadafc9 was re-scheduled: Binding failed for port f0eb55e9-0018-4c0e-99cb-227551afe2c8, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 870.221345] env[62499]: DEBUG nova.compute.manager [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 870.221930] env[62499]: DEBUG oslo_concurrency.lockutils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "refresh_cache-263931cd-b2dc-41bb-8a2b-abf61aadafc9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 870.221930] env[62499]: DEBUG oslo_concurrency.lockutils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquired lock "refresh_cache-263931cd-b2dc-41bb-8a2b-abf61aadafc9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 870.221930] env[62499]: DEBUG nova.network.neutron [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 870.238394] env[62499]: DEBUG nova.compute.manager [req-4bdcd1a9-7fca-4aeb-839a-3a8b1bce46f8 req-4b61f20e-14d7-42dc-8ee1-5a879a520c10 service nova] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Received event network-changed-296e37b0-c4ab-4591-9b5f-d51c80bb0f9c {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 870.238394] env[62499]: DEBUG nova.compute.manager [req-4bdcd1a9-7fca-4aeb-839a-3a8b1bce46f8 req-4b61f20e-14d7-42dc-8ee1-5a879a520c10 service nova] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Refreshing instance network info cache due to event network-changed-296e37b0-c4ab-4591-9b5f-d51c80bb0f9c. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 870.238394] env[62499]: DEBUG oslo_concurrency.lockutils [req-4bdcd1a9-7fca-4aeb-839a-3a8b1bce46f8 req-4b61f20e-14d7-42dc-8ee1-5a879a520c10 service nova] Acquiring lock "refresh_cache-c1961f60-06d8-44d8-999f-d498c3667adc" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 870.238394] env[62499]: DEBUG oslo_concurrency.lockutils [req-4bdcd1a9-7fca-4aeb-839a-3a8b1bce46f8 req-4b61f20e-14d7-42dc-8ee1-5a879a520c10 service nova] Acquired lock "refresh_cache-c1961f60-06d8-44d8-999f-d498c3667adc" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 870.238394] env[62499]: DEBUG nova.network.neutron [req-4bdcd1a9-7fca-4aeb-839a-3a8b1bce46f8 req-4b61f20e-14d7-42dc-8ee1-5a879a520c10 service nova] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Refreshing network info cache for port 296e37b0-c4ab-4591-9b5f-d51c80bb0f9c {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 870.257947] env[62499]: DEBUG nova.compute.manager [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 870.416703] env[62499]: DEBUG nova.compute.manager [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 870.437241] env[62499]: ERROR nova.compute.manager [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 296e37b0-c4ab-4591-9b5f-d51c80bb0f9c, please check neutron logs for more information. [ 870.437241] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 870.437241] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 870.437241] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 870.437241] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 870.437241] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 870.437241] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 870.437241] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 870.437241] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 870.437241] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 870.437241] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 870.437241] env[62499]: ERROR nova.compute.manager raise self.value [ 870.437241] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 870.437241] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 870.437241] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 870.437241] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 870.437880] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 870.437880] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 870.437880] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 296e37b0-c4ab-4591-9b5f-d51c80bb0f9c, please check neutron logs for more information. [ 870.437880] env[62499]: ERROR nova.compute.manager [ 870.437880] env[62499]: Traceback (most recent call last): [ 870.437880] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 870.437880] env[62499]: listener.cb(fileno) [ 870.437880] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 870.437880] env[62499]: result = function(*args, **kwargs) [ 870.437880] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 870.437880] env[62499]: return func(*args, **kwargs) [ 870.437880] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 870.437880] env[62499]: raise e [ 870.437880] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 870.437880] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 870.437880] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 870.437880] env[62499]: created_port_ids = self._update_ports_for_instance( [ 870.437880] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 870.437880] env[62499]: with excutils.save_and_reraise_exception(): [ 870.437880] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 870.437880] env[62499]: self.force_reraise() [ 870.437880] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 870.437880] env[62499]: raise self.value [ 870.437880] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 870.437880] env[62499]: updated_port = self._update_port( [ 870.437880] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 870.437880] env[62499]: _ensure_no_port_binding_failure(port) [ 870.437880] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 870.437880] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 870.439361] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 296e37b0-c4ab-4591-9b5f-d51c80bb0f9c, please check neutron logs for more information. [ 870.439361] env[62499]: Removing descriptor: 16 [ 870.448451] env[62499]: DEBUG nova.virt.hardware [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 870.448635] env[62499]: DEBUG nova.virt.hardware [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 870.448768] env[62499]: DEBUG nova.virt.hardware [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 870.448950] env[62499]: DEBUG nova.virt.hardware [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 870.449119] env[62499]: DEBUG nova.virt.hardware [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 870.449379] env[62499]: DEBUG nova.virt.hardware [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 870.449454] env[62499]: DEBUG nova.virt.hardware [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 870.449698] env[62499]: DEBUG nova.virt.hardware [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 870.449773] env[62499]: DEBUG nova.virt.hardware [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 870.449925] env[62499]: DEBUG nova.virt.hardware [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 870.450258] env[62499]: DEBUG nova.virt.hardware [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 870.451183] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1ebb152-6d17-42ad-969b-d1d42dde1a2f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.460212] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e6248300-7a4d-4843-aa9d-86141d9f4ea1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.475489] env[62499]: ERROR nova.compute.manager [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 296e37b0-c4ab-4591-9b5f-d51c80bb0f9c, please check neutron logs for more information. [ 870.475489] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Traceback (most recent call last): [ 870.475489] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 870.475489] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] yield resources [ 870.475489] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 870.475489] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] self.driver.spawn(context, instance, image_meta, [ 870.475489] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 870.475489] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] self._vmops.spawn(context, instance, image_meta, injected_files, [ 870.475489] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 870.475489] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] vm_ref = self.build_virtual_machine(instance, [ 870.475489] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 870.475948] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] vif_infos = vmwarevif.get_vif_info(self._session, [ 870.475948] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 870.475948] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] for vif in network_info: [ 870.475948] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 870.475948] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] return self._sync_wrapper(fn, *args, **kwargs) [ 870.475948] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 870.475948] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] self.wait() [ 870.475948] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 870.475948] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] self[:] = self._gt.wait() [ 870.475948] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 870.475948] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] return self._exit_event.wait() [ 870.475948] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 870.475948] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] current.throw(*self._exc) [ 870.476405] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 870.476405] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] result = function(*args, **kwargs) [ 870.476405] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 870.476405] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] return func(*args, **kwargs) [ 870.476405] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 870.476405] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] raise e [ 870.476405] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 870.476405] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] nwinfo = self.network_api.allocate_for_instance( [ 870.476405] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 870.476405] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] created_port_ids = self._update_ports_for_instance( [ 870.476405] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 870.476405] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] with excutils.save_and_reraise_exception(): [ 870.476405] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 870.476834] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] self.force_reraise() [ 870.476834] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 870.476834] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] raise self.value [ 870.476834] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 870.476834] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] updated_port = self._update_port( [ 870.476834] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 870.476834] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] _ensure_no_port_binding_failure(port) [ 870.476834] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 870.476834] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] raise exception.PortBindingFailed(port_id=port['id']) [ 870.476834] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] nova.exception.PortBindingFailed: Binding failed for port 296e37b0-c4ab-4591-9b5f-d51c80bb0f9c, please check neutron logs for more information. [ 870.476834] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] [ 870.476834] env[62499]: INFO nova.compute.manager [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Terminating instance [ 870.477798] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquiring lock "refresh_cache-c1961f60-06d8-44d8-999f-d498c3667adc" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 870.752775] env[62499]: DEBUG nova.network.neutron [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 870.759074] env[62499]: DEBUG nova.network.neutron [req-4bdcd1a9-7fca-4aeb-839a-3a8b1bce46f8 req-4b61f20e-14d7-42dc-8ee1-5a879a520c10 service nova] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 870.779574] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 870.858979] env[62499]: DEBUG nova.network.neutron [req-4bdcd1a9-7fca-4aeb-839a-3a8b1bce46f8 req-4b61f20e-14d7-42dc-8ee1-5a879a520c10 service nova] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 870.951519] env[62499]: DEBUG nova.network.neutron [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 870.958883] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2ad4023-44d3-418e-9128-00ad7b9dad98 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.970348] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3f26c26-975b-478c-9712-ddc373512662 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 871.004547] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e1eb0ec2-cc97-4faa-9999-aa4950d7dd9f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 871.012355] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b49537bf-4201-41ee-99ce-45b7c79f2414 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 871.026020] env[62499]: DEBUG nova.compute.provider_tree [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 871.362805] env[62499]: DEBUG oslo_concurrency.lockutils [req-4bdcd1a9-7fca-4aeb-839a-3a8b1bce46f8 req-4b61f20e-14d7-42dc-8ee1-5a879a520c10 service nova] Releasing lock "refresh_cache-c1961f60-06d8-44d8-999f-d498c3667adc" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 871.363113] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquired lock "refresh_cache-c1961f60-06d8-44d8-999f-d498c3667adc" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 871.363204] env[62499]: DEBUG nova.network.neutron [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 871.457659] env[62499]: DEBUG oslo_concurrency.lockutils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Releasing lock "refresh_cache-263931cd-b2dc-41bb-8a2b-abf61aadafc9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 871.458521] env[62499]: DEBUG nova.compute.manager [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 871.458521] env[62499]: DEBUG nova.compute.manager [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 871.458521] env[62499]: DEBUG nova.network.neutron [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 871.489029] env[62499]: DEBUG nova.network.neutron [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 871.528282] env[62499]: DEBUG nova.scheduler.client.report [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 871.972291] env[62499]: DEBUG nova.network.neutron [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 871.991510] env[62499]: DEBUG nova.network.neutron [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 872.037211] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.817s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 872.037211] env[62499]: ERROR nova.compute.manager [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 588d004d-9a19-4de8-9e31-22dc0aeede5b, please check neutron logs for more information. [ 872.037211] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Traceback (most recent call last): [ 872.037211] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 872.037211] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] self.driver.spawn(context, instance, image_meta, [ 872.037211] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 872.037211] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] self._vmops.spawn(context, instance, image_meta, injected_files, [ 872.037211] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 872.037211] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] vm_ref = self.build_virtual_machine(instance, [ 872.037519] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 872.037519] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] vif_infos = vmwarevif.get_vif_info(self._session, [ 872.037519] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 872.037519] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] for vif in network_info: [ 872.037519] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 872.037519] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] return self._sync_wrapper(fn, *args, **kwargs) [ 872.037519] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 872.037519] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] self.wait() [ 872.037519] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 872.037519] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] self[:] = self._gt.wait() [ 872.037519] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 872.037519] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] return self._exit_event.wait() [ 872.037519] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 872.037826] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] current.throw(*self._exc) [ 872.037826] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 872.037826] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] result = function(*args, **kwargs) [ 872.037826] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 872.037826] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] return func(*args, **kwargs) [ 872.037826] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 872.037826] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] raise e [ 872.037826] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 872.037826] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] nwinfo = self.network_api.allocate_for_instance( [ 872.037826] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 872.037826] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] created_port_ids = self._update_ports_for_instance( [ 872.037826] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 872.037826] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] with excutils.save_and_reraise_exception(): [ 872.038183] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 872.038183] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] self.force_reraise() [ 872.038183] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 872.038183] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] raise self.value [ 872.038183] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 872.038183] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] updated_port = self._update_port( [ 872.038183] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 872.038183] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] _ensure_no_port_binding_failure(port) [ 872.038183] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 872.038183] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] raise exception.PortBindingFailed(port_id=port['id']) [ 872.038183] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] nova.exception.PortBindingFailed: Binding failed for port 588d004d-9a19-4de8-9e31-22dc0aeede5b, please check neutron logs for more information. [ 872.038183] env[62499]: ERROR nova.compute.manager [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] [ 872.038479] env[62499]: DEBUG nova.compute.utils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Binding failed for port 588d004d-9a19-4de8-9e31-22dc0aeede5b, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 872.038940] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.744s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 872.044917] env[62499]: DEBUG nova.compute.manager [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Build of instance 20e4ff1b-0b84-477d-a0d0-a85a439449ad was re-scheduled: Binding failed for port 588d004d-9a19-4de8-9e31-22dc0aeede5b, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 872.044917] env[62499]: DEBUG nova.compute.manager [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 872.044917] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Acquiring lock "refresh_cache-20e4ff1b-0b84-477d-a0d0-a85a439449ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 872.044917] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Acquired lock "refresh_cache-20e4ff1b-0b84-477d-a0d0-a85a439449ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 872.045329] env[62499]: DEBUG nova.network.neutron [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 872.121510] env[62499]: DEBUG nova.network.neutron [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 872.268106] env[62499]: DEBUG nova.compute.manager [req-f5d65dc2-073c-44cd-912d-daee99f20666 req-bd556a5f-3aa6-4245-8c13-26c3fef0d72d service nova] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Received event network-vif-deleted-296e37b0-c4ab-4591-9b5f-d51c80bb0f9c {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 872.495211] env[62499]: INFO nova.compute.manager [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 263931cd-b2dc-41bb-8a2b-abf61aadafc9] Took 1.04 seconds to deallocate network for instance. [ 872.585026] env[62499]: DEBUG nova.network.neutron [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 872.622900] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Releasing lock "refresh_cache-c1961f60-06d8-44d8-999f-d498c3667adc" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 872.623350] env[62499]: DEBUG nova.compute.manager [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 872.623546] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 872.623838] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-13967e97-e38e-40d4-aedc-ac6350309a0c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.638493] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b453fc4a-9224-409b-a31a-4a683a39203f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.666622] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c1961f60-06d8-44d8-999f-d498c3667adc could not be found. [ 872.666622] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 872.666622] env[62499]: INFO nova.compute.manager [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Took 0.04 seconds to destroy the instance on the hypervisor. [ 872.667041] env[62499]: DEBUG oslo.service.loopingcall [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 872.668211] env[62499]: DEBUG nova.compute.manager [-] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 872.668211] env[62499]: DEBUG nova.network.neutron [-] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 872.697639] env[62499]: DEBUG nova.network.neutron [-] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 872.737984] env[62499]: DEBUG nova.network.neutron [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 872.836923] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-450b56bb-6ec5-4679-9e6d-4f53b47ec450 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.851887] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-908021a7-71ba-40eb-9c0c-4eee791d9c19 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.884787] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54d62da9-9382-41cc-a905-023376844882 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.892208] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0db4603a-9a8b-484b-a8b8-37be81eefc57 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.906093] env[62499]: DEBUG nova.compute.provider_tree [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 873.202187] env[62499]: DEBUG nova.network.neutron [-] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 873.244944] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Releasing lock "refresh_cache-20e4ff1b-0b84-477d-a0d0-a85a439449ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 873.244944] env[62499]: DEBUG nova.compute.manager [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 873.244944] env[62499]: DEBUG nova.compute.manager [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 873.244944] env[62499]: DEBUG nova.network.neutron [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 873.274062] env[62499]: DEBUG nova.network.neutron [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 873.409551] env[62499]: DEBUG nova.scheduler.client.report [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 873.532859] env[62499]: INFO nova.scheduler.client.report [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Deleted allocations for instance 263931cd-b2dc-41bb-8a2b-abf61aadafc9 [ 873.704364] env[62499]: INFO nova.compute.manager [-] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Took 1.04 seconds to deallocate network for instance. [ 873.710295] env[62499]: DEBUG nova.compute.claims [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 873.710295] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 873.778289] env[62499]: DEBUG nova.network.neutron [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 873.915050] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.877s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 873.915801] env[62499]: ERROR nova.compute.manager [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port dee6d343-e56a-4d3a-9f5f-2a6fe259825e, please check neutron logs for more information. [ 873.915801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Traceback (most recent call last): [ 873.915801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 873.915801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] self.driver.spawn(context, instance, image_meta, [ 873.915801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 873.915801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] self._vmops.spawn(context, instance, image_meta, injected_files, [ 873.915801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 873.915801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] vm_ref = self.build_virtual_machine(instance, [ 873.915801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 873.915801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] vif_infos = vmwarevif.get_vif_info(self._session, [ 873.915801] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 873.916117] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] for vif in network_info: [ 873.916117] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 873.916117] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] return self._sync_wrapper(fn, *args, **kwargs) [ 873.916117] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 873.916117] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] self.wait() [ 873.916117] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 873.916117] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] self[:] = self._gt.wait() [ 873.916117] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 873.916117] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] return self._exit_event.wait() [ 873.916117] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 873.916117] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] current.throw(*self._exc) [ 873.916117] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 873.916117] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] result = function(*args, **kwargs) [ 873.916439] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 873.916439] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] return func(*args, **kwargs) [ 873.916439] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 873.916439] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] raise e [ 873.916439] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 873.916439] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] nwinfo = self.network_api.allocate_for_instance( [ 873.916439] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 873.916439] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] created_port_ids = self._update_ports_for_instance( [ 873.916439] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 873.916439] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] with excutils.save_and_reraise_exception(): [ 873.916439] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 873.916439] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] self.force_reraise() [ 873.916439] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 873.916806] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] raise self.value [ 873.916806] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 873.916806] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] updated_port = self._update_port( [ 873.916806] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 873.916806] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] _ensure_no_port_binding_failure(port) [ 873.916806] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 873.916806] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] raise exception.PortBindingFailed(port_id=port['id']) [ 873.916806] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] nova.exception.PortBindingFailed: Binding failed for port dee6d343-e56a-4d3a-9f5f-2a6fe259825e, please check neutron logs for more information. [ 873.916806] env[62499]: ERROR nova.compute.manager [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] [ 873.916806] env[62499]: DEBUG nova.compute.utils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Binding failed for port dee6d343-e56a-4d3a-9f5f-2a6fe259825e, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 873.918994] env[62499]: DEBUG nova.compute.manager [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Build of instance 4eda80db-5db7-4fd6-9932-108c22f4a616 was re-scheduled: Binding failed for port dee6d343-e56a-4d3a-9f5f-2a6fe259825e, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 873.919206] env[62499]: DEBUG nova.compute.manager [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 873.919290] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Acquiring lock "refresh_cache-4eda80db-5db7-4fd6-9932-108c22f4a616" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 873.919445] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Acquired lock "refresh_cache-4eda80db-5db7-4fd6-9932-108c22f4a616" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 873.919609] env[62499]: DEBUG nova.network.neutron [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 873.921138] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 17.586s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 873.921486] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 873.921688] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62499) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 873.922161] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 14.389s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 873.922395] env[62499]: DEBUG nova.objects.instance [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lazy-loading 'resources' on Instance uuid 008b230e-72b8-43ae-826a-b38111c56e76 {{(pid=62499) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 873.924157] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2bb9ebe5-aaf0-49c6-b589-280fd4036599 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 873.936663] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20984ee7-00e2-494e-8d34-fa70ccfd210d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 873.953159] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e68b676e-50f2-4473-b332-8871d028b536 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 873.960206] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-afc06cb3-c992-45bd-8aa9-141f70bbb8d0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 873.993141] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181258MB free_disk=130GB free_vcpus=48 pci_devices=None {{(pid=62499) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 873.993141] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 874.044061] env[62499]: DEBUG oslo_concurrency.lockutils [None req-aa5be995-82c9-4e5c-b672-c3cd63dccbe5 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "263931cd-b2dc-41bb-8a2b-abf61aadafc9" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 138.605s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 874.241796] env[62499]: DEBUG oslo_concurrency.lockutils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Acquiring lock "4af6c7f4-b774-4c3f-80af-e8704a904c38" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 874.242032] env[62499]: DEBUG oslo_concurrency.lockutils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Lock "4af6c7f4-b774-4c3f-80af-e8704a904c38" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 874.284139] env[62499]: INFO nova.compute.manager [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] [instance: 20e4ff1b-0b84-477d-a0d0-a85a439449ad] Took 1.04 seconds to deallocate network for instance. [ 874.443931] env[62499]: DEBUG nova.network.neutron [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 874.548633] env[62499]: DEBUG nova.compute.manager [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 874.607199] env[62499]: DEBUG nova.network.neutron [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 874.636441] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-208f14d8-25c5-4b6e-86e8-c862e6c11df5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 874.644778] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99d1f4f8-d536-44a9-a2e0-ddf2d68c7dc6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 874.677638] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5eb51439-82a6-4a6b-8b5c-3b1ec7697bbd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 874.684921] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-292e0614-6b42-4c08-bd69-a9789bc7402a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 874.700021] env[62499]: DEBUG nova.compute.provider_tree [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 874.958741] env[62499]: DEBUG oslo_concurrency.lockutils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "4fa63411-a721-4e34-a08d-90587b0b34d3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 874.959140] env[62499]: DEBUG oslo_concurrency.lockutils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "4fa63411-a721-4e34-a08d-90587b0b34d3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 875.072708] env[62499]: DEBUG oslo_concurrency.lockutils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 875.114017] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Releasing lock "refresh_cache-4eda80db-5db7-4fd6-9932-108c22f4a616" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 875.114282] env[62499]: DEBUG nova.compute.manager [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 875.114475] env[62499]: DEBUG nova.compute.manager [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 875.114645] env[62499]: DEBUG nova.network.neutron [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 875.132128] env[62499]: DEBUG nova.network.neutron [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 875.202400] env[62499]: DEBUG nova.scheduler.client.report [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 875.315837] env[62499]: INFO nova.scheduler.client.report [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Deleted allocations for instance 20e4ff1b-0b84-477d-a0d0-a85a439449ad [ 875.634571] env[62499]: DEBUG nova.network.neutron [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 875.716697] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.794s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 875.719033] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.135s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 875.720015] env[62499]: INFO nova.compute.claims [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 875.737048] env[62499]: INFO nova.scheduler.client.report [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Deleted allocations for instance 008b230e-72b8-43ae-826a-b38111c56e76 [ 875.824463] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d5fffe0e-8595-4212-b847-9f4fb17f5bc9 tempest-ServersNegativeTestMultiTenantJSON-1317981700 tempest-ServersNegativeTestMultiTenantJSON-1317981700-project-member] Lock "20e4ff1b-0b84-477d-a0d0-a85a439449ad" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 133.972s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 876.137401] env[62499]: INFO nova.compute.manager [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 4eda80db-5db7-4fd6-9932-108c22f4a616] Took 1.02 seconds to deallocate network for instance. [ 876.245283] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b98d3dc6-350d-4dbc-93ab-a528e106dba0 tempest-ServerShowV247Test-575892637 tempest-ServerShowV247Test-575892637-project-member] Lock "008b230e-72b8-43ae-826a-b38111c56e76" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 22.106s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 876.328115] env[62499]: DEBUG nova.compute.manager [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 876.720017] env[62499]: DEBUG oslo_concurrency.lockutils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Acquiring lock "88466440-7392-43d3-83ec-45229516a3ef" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 876.720357] env[62499]: DEBUG oslo_concurrency.lockutils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Lock "88466440-7392-43d3-83ec-45229516a3ef" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 876.853148] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 876.966924] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a687ec46-b628-4128-a9de-3a461aebc726 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 876.976279] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5762fb47-7ac8-4c25-9f6c-4b6d79005708 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 877.008783] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2580170-263f-4a7f-b615-d55bc55f6c99 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 877.016760] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c3994c9-6dec-446e-95b7-c94fa760214e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 877.032340] env[62499]: DEBUG nova.compute.provider_tree [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 877.171131] env[62499]: INFO nova.scheduler.client.report [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Deleted allocations for instance 4eda80db-5db7-4fd6-9932-108c22f4a616 [ 877.536192] env[62499]: DEBUG nova.scheduler.client.report [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 877.681230] env[62499]: DEBUG oslo_concurrency.lockutils [None req-3bd4fa14-cb89-4456-85a7-187db1076363 tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Lock "4eda80db-5db7-4fd6-9932-108c22f4a616" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 130.839s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 878.047966] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.329s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 878.048545] env[62499]: DEBUG nova.compute.manager [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 878.051150] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.750s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 878.052521] env[62499]: INFO nova.compute.claims [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 878.185299] env[62499]: DEBUG nova.compute.manager [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 878.556979] env[62499]: DEBUG nova.compute.utils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 878.558779] env[62499]: DEBUG nova.compute.manager [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 878.558986] env[62499]: DEBUG nova.network.neutron [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 878.628717] env[62499]: DEBUG nova.policy [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd67d200795f843cc81b10074a30aae4a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9056b5a2acfa45a48b5b1530dd9b7860', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 878.712139] env[62499]: DEBUG oslo_concurrency.lockutils [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 878.893065] env[62499]: DEBUG nova.network.neutron [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Successfully created port: 2a8fbacb-6166-43d0-b9f4-1c05df641aad {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 879.063010] env[62499]: DEBUG nova.compute.manager [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 879.307292] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a1da5bd7-35c9-4e2e-b1a4-8dd454ea0a51 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 879.315581] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b026690a-97e8-4019-aa99-ba7ac3a45f70 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 879.348220] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e4341ab-6e9b-45fa-bc4c-f532c24a950c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 879.356979] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0892eca2-4103-4a3c-bd90-152c20b1a6ee {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 879.373178] env[62499]: DEBUG nova.compute.provider_tree [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 879.876740] env[62499]: DEBUG nova.scheduler.client.report [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 880.045244] env[62499]: DEBUG nova.compute.manager [req-4b789544-b486-406c-9d65-95e27a594f2a req-6adcd6e5-3bee-44fa-975b-6e0ea9270656 service nova] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Received event network-changed-2a8fbacb-6166-43d0-b9f4-1c05df641aad {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 880.045469] env[62499]: DEBUG nova.compute.manager [req-4b789544-b486-406c-9d65-95e27a594f2a req-6adcd6e5-3bee-44fa-975b-6e0ea9270656 service nova] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Refreshing instance network info cache due to event network-changed-2a8fbacb-6166-43d0-b9f4-1c05df641aad. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 880.045690] env[62499]: DEBUG oslo_concurrency.lockutils [req-4b789544-b486-406c-9d65-95e27a594f2a req-6adcd6e5-3bee-44fa-975b-6e0ea9270656 service nova] Acquiring lock "refresh_cache-ba8fc3e0-dae8-4910-858b-dab1de72089e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 880.045833] env[62499]: DEBUG oslo_concurrency.lockutils [req-4b789544-b486-406c-9d65-95e27a594f2a req-6adcd6e5-3bee-44fa-975b-6e0ea9270656 service nova] Acquired lock "refresh_cache-ba8fc3e0-dae8-4910-858b-dab1de72089e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 880.045991] env[62499]: DEBUG nova.network.neutron [req-4b789544-b486-406c-9d65-95e27a594f2a req-6adcd6e5-3bee-44fa-975b-6e0ea9270656 service nova] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Refreshing network info cache for port 2a8fbacb-6166-43d0-b9f4-1c05df641aad {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 880.073393] env[62499]: DEBUG nova.compute.manager [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 880.097909] env[62499]: DEBUG nova.virt.hardware [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 880.098171] env[62499]: DEBUG nova.virt.hardware [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 880.098329] env[62499]: DEBUG nova.virt.hardware [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 880.098509] env[62499]: DEBUG nova.virt.hardware [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 880.098685] env[62499]: DEBUG nova.virt.hardware [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 880.098794] env[62499]: DEBUG nova.virt.hardware [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 880.098994] env[62499]: DEBUG nova.virt.hardware [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 880.099257] env[62499]: DEBUG nova.virt.hardware [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 880.099433] env[62499]: DEBUG nova.virt.hardware [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 880.099602] env[62499]: DEBUG nova.virt.hardware [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 880.099767] env[62499]: DEBUG nova.virt.hardware [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 880.100654] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2309a924-e281-48d8-a915-d84b1d55b082 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 880.109772] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5046dd5-7345-4824-85de-7af1ccf28d8a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 880.260728] env[62499]: ERROR nova.compute.manager [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 2a8fbacb-6166-43d0-b9f4-1c05df641aad, please check neutron logs for more information. [ 880.260728] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 880.260728] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 880.260728] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 880.260728] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 880.260728] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 880.260728] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 880.260728] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 880.260728] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 880.260728] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 880.260728] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 880.260728] env[62499]: ERROR nova.compute.manager raise self.value [ 880.260728] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 880.260728] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 880.260728] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 880.260728] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 880.262457] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 880.262457] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 880.262457] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 2a8fbacb-6166-43d0-b9f4-1c05df641aad, please check neutron logs for more information. [ 880.262457] env[62499]: ERROR nova.compute.manager [ 880.262457] env[62499]: Traceback (most recent call last): [ 880.262457] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 880.262457] env[62499]: listener.cb(fileno) [ 880.262457] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 880.262457] env[62499]: result = function(*args, **kwargs) [ 880.262457] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 880.262457] env[62499]: return func(*args, **kwargs) [ 880.262457] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 880.262457] env[62499]: raise e [ 880.262457] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 880.262457] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 880.262457] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 880.262457] env[62499]: created_port_ids = self._update_ports_for_instance( [ 880.262457] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 880.262457] env[62499]: with excutils.save_and_reraise_exception(): [ 880.262457] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 880.262457] env[62499]: self.force_reraise() [ 880.262457] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 880.262457] env[62499]: raise self.value [ 880.262457] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 880.262457] env[62499]: updated_port = self._update_port( [ 880.262457] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 880.262457] env[62499]: _ensure_no_port_binding_failure(port) [ 880.262457] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 880.262457] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 880.263499] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 2a8fbacb-6166-43d0-b9f4-1c05df641aad, please check neutron logs for more information. [ 880.263499] env[62499]: Removing descriptor: 16 [ 880.263499] env[62499]: ERROR nova.compute.manager [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 2a8fbacb-6166-43d0-b9f4-1c05df641aad, please check neutron logs for more information. [ 880.263499] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Traceback (most recent call last): [ 880.263499] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 880.263499] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] yield resources [ 880.263499] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 880.263499] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] self.driver.spawn(context, instance, image_meta, [ 880.263499] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 880.263499] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 880.263499] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 880.263499] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] vm_ref = self.build_virtual_machine(instance, [ 880.263788] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 880.263788] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] vif_infos = vmwarevif.get_vif_info(self._session, [ 880.263788] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 880.263788] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] for vif in network_info: [ 880.263788] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 880.263788] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] return self._sync_wrapper(fn, *args, **kwargs) [ 880.263788] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 880.263788] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] self.wait() [ 880.263788] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 880.263788] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] self[:] = self._gt.wait() [ 880.263788] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 880.263788] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] return self._exit_event.wait() [ 880.263788] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 880.265429] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] result = hub.switch() [ 880.265429] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 880.265429] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] return self.greenlet.switch() [ 880.265429] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 880.265429] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] result = function(*args, **kwargs) [ 880.265429] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 880.265429] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] return func(*args, **kwargs) [ 880.265429] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 880.265429] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] raise e [ 880.265429] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 880.265429] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] nwinfo = self.network_api.allocate_for_instance( [ 880.265429] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 880.265429] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] created_port_ids = self._update_ports_for_instance( [ 880.270048] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 880.270048] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] with excutils.save_and_reraise_exception(): [ 880.270048] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 880.270048] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] self.force_reraise() [ 880.270048] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 880.270048] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] raise self.value [ 880.270048] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 880.270048] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] updated_port = self._update_port( [ 880.270048] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 880.270048] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] _ensure_no_port_binding_failure(port) [ 880.270048] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 880.270048] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] raise exception.PortBindingFailed(port_id=port['id']) [ 880.270447] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] nova.exception.PortBindingFailed: Binding failed for port 2a8fbacb-6166-43d0-b9f4-1c05df641aad, please check neutron logs for more information. [ 880.270447] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] [ 880.270447] env[62499]: INFO nova.compute.manager [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Terminating instance [ 880.270447] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquiring lock "refresh_cache-ba8fc3e0-dae8-4910-858b-dab1de72089e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 880.386094] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.335s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 880.386654] env[62499]: DEBUG nova.compute.manager [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 880.389212] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.319s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 880.390715] env[62499]: INFO nova.compute.claims [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 880.568507] env[62499]: DEBUG nova.network.neutron [req-4b789544-b486-406c-9d65-95e27a594f2a req-6adcd6e5-3bee-44fa-975b-6e0ea9270656 service nova] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 880.650305] env[62499]: DEBUG nova.network.neutron [req-4b789544-b486-406c-9d65-95e27a594f2a req-6adcd6e5-3bee-44fa-975b-6e0ea9270656 service nova] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 880.788743] env[62499]: DEBUG oslo_concurrency.lockutils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Acquiring lock "95a5cb2c-24da-43c0-b9d1-7f0b51ccda87" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 880.788948] env[62499]: DEBUG oslo_concurrency.lockutils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Lock "95a5cb2c-24da-43c0-b9d1-7f0b51ccda87" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 880.894743] env[62499]: DEBUG nova.compute.utils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 880.897996] env[62499]: DEBUG nova.compute.manager [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 880.898889] env[62499]: DEBUG nova.network.neutron [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 880.947972] env[62499]: DEBUG nova.policy [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd67d200795f843cc81b10074a30aae4a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9056b5a2acfa45a48b5b1530dd9b7860', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 881.153778] env[62499]: DEBUG oslo_concurrency.lockutils [req-4b789544-b486-406c-9d65-95e27a594f2a req-6adcd6e5-3bee-44fa-975b-6e0ea9270656 service nova] Releasing lock "refresh_cache-ba8fc3e0-dae8-4910-858b-dab1de72089e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 881.154268] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquired lock "refresh_cache-ba8fc3e0-dae8-4910-858b-dab1de72089e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 881.154504] env[62499]: DEBUG nova.network.neutron [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 881.195611] env[62499]: DEBUG nova.network.neutron [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Successfully created port: 4681981e-8791-4c18-b0e8-49b24f59baa0 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 881.405018] env[62499]: DEBUG nova.compute.manager [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 881.617944] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da7e7f7a-7ec8-495e-b812-a3ff40df5a2e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 881.625737] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-43807ef3-dc4f-4fab-bf94-8643feec6eb2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 881.655186] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c39bcb3-4808-4de1-8a0d-6e7531dfe412 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 881.664107] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4cdfce2f-cf81-4665-8abe-6700c38c4f31 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 881.678034] env[62499]: DEBUG nova.compute.provider_tree [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 881.683026] env[62499]: DEBUG nova.network.neutron [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 881.759483] env[62499]: DEBUG nova.network.neutron [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 882.043577] env[62499]: ERROR nova.compute.manager [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 4681981e-8791-4c18-b0e8-49b24f59baa0, please check neutron logs for more information. [ 882.043577] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 882.043577] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 882.043577] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 882.043577] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 882.043577] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 882.043577] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 882.043577] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 882.043577] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 882.043577] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 882.043577] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 882.043577] env[62499]: ERROR nova.compute.manager raise self.value [ 882.043577] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 882.043577] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 882.043577] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 882.043577] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 882.044074] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 882.044074] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 882.044074] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 4681981e-8791-4c18-b0e8-49b24f59baa0, please check neutron logs for more information. [ 882.044074] env[62499]: ERROR nova.compute.manager [ 882.044074] env[62499]: Traceback (most recent call last): [ 882.044074] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 882.044074] env[62499]: listener.cb(fileno) [ 882.044074] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 882.044074] env[62499]: result = function(*args, **kwargs) [ 882.044074] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 882.044074] env[62499]: return func(*args, **kwargs) [ 882.044074] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 882.044074] env[62499]: raise e [ 882.044074] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 882.044074] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 882.044074] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 882.044074] env[62499]: created_port_ids = self._update_ports_for_instance( [ 882.044074] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 882.044074] env[62499]: with excutils.save_and_reraise_exception(): [ 882.044074] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 882.044074] env[62499]: self.force_reraise() [ 882.044074] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 882.044074] env[62499]: raise self.value [ 882.044074] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 882.044074] env[62499]: updated_port = self._update_port( [ 882.044074] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 882.044074] env[62499]: _ensure_no_port_binding_failure(port) [ 882.044074] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 882.044074] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 882.044882] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 4681981e-8791-4c18-b0e8-49b24f59baa0, please check neutron logs for more information. [ 882.044882] env[62499]: Removing descriptor: 16 [ 882.075898] env[62499]: DEBUG nova.compute.manager [req-a6ce2bd3-dbac-491c-93fc-edf63ed4309e req-3121f220-94cf-4493-afb8-ad267bd99eb8 service nova] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Received event network-vif-deleted-2a8fbacb-6166-43d0-b9f4-1c05df641aad {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 882.076071] env[62499]: DEBUG nova.compute.manager [req-a6ce2bd3-dbac-491c-93fc-edf63ed4309e req-3121f220-94cf-4493-afb8-ad267bd99eb8 service nova] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Received event network-changed-4681981e-8791-4c18-b0e8-49b24f59baa0 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 882.076235] env[62499]: DEBUG nova.compute.manager [req-a6ce2bd3-dbac-491c-93fc-edf63ed4309e req-3121f220-94cf-4493-afb8-ad267bd99eb8 service nova] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Refreshing instance network info cache due to event network-changed-4681981e-8791-4c18-b0e8-49b24f59baa0. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 882.076440] env[62499]: DEBUG oslo_concurrency.lockutils [req-a6ce2bd3-dbac-491c-93fc-edf63ed4309e req-3121f220-94cf-4493-afb8-ad267bd99eb8 service nova] Acquiring lock "refresh_cache-0eaadb76-e2c7-4e06-b865-065ab0f014ba" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 882.076578] env[62499]: DEBUG oslo_concurrency.lockutils [req-a6ce2bd3-dbac-491c-93fc-edf63ed4309e req-3121f220-94cf-4493-afb8-ad267bd99eb8 service nova] Acquired lock "refresh_cache-0eaadb76-e2c7-4e06-b865-065ab0f014ba" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 882.076734] env[62499]: DEBUG nova.network.neutron [req-a6ce2bd3-dbac-491c-93fc-edf63ed4309e req-3121f220-94cf-4493-afb8-ad267bd99eb8 service nova] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Refreshing network info cache for port 4681981e-8791-4c18-b0e8-49b24f59baa0 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 882.183402] env[62499]: DEBUG nova.scheduler.client.report [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 882.261584] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Releasing lock "refresh_cache-ba8fc3e0-dae8-4910-858b-dab1de72089e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 882.261999] env[62499]: DEBUG nova.compute.manager [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 882.262215] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 882.262495] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2f026c83-07e8-4b79-ad9b-e40dd5a93ed8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 882.271754] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e724b208-fa80-4c43-b81e-237a2ca00576 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 882.291542] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ba8fc3e0-dae8-4910-858b-dab1de72089e could not be found. [ 882.291740] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 882.291918] env[62499]: INFO nova.compute.manager [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Took 0.03 seconds to destroy the instance on the hypervisor. [ 882.292155] env[62499]: DEBUG oslo.service.loopingcall [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 882.292357] env[62499]: DEBUG nova.compute.manager [-] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 882.292448] env[62499]: DEBUG nova.network.neutron [-] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 882.308140] env[62499]: DEBUG nova.network.neutron [-] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 882.414132] env[62499]: DEBUG nova.compute.manager [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 882.440231] env[62499]: DEBUG nova.virt.hardware [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:39Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='84',id=12,is_public=True,memory_mb=256,name='m1.micro',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 882.440492] env[62499]: DEBUG nova.virt.hardware [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 882.440648] env[62499]: DEBUG nova.virt.hardware [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 882.440825] env[62499]: DEBUG nova.virt.hardware [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 882.440997] env[62499]: DEBUG nova.virt.hardware [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 882.441175] env[62499]: DEBUG nova.virt.hardware [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 882.441379] env[62499]: DEBUG nova.virt.hardware [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 882.441531] env[62499]: DEBUG nova.virt.hardware [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 882.441689] env[62499]: DEBUG nova.virt.hardware [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 882.441855] env[62499]: DEBUG nova.virt.hardware [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 882.442354] env[62499]: DEBUG nova.virt.hardware [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 882.442882] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fbb554b7-d937-4755-9856-6bdebc0616ca {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 882.452794] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f42c446-1c7d-494d-a3d9-58e9076189fd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 882.466547] env[62499]: ERROR nova.compute.manager [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 4681981e-8791-4c18-b0e8-49b24f59baa0, please check neutron logs for more information. [ 882.466547] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Traceback (most recent call last): [ 882.466547] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 882.466547] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] yield resources [ 882.466547] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 882.466547] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] self.driver.spawn(context, instance, image_meta, [ 882.466547] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 882.466547] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] self._vmops.spawn(context, instance, image_meta, injected_files, [ 882.466547] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 882.466547] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] vm_ref = self.build_virtual_machine(instance, [ 882.466547] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 882.466866] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] vif_infos = vmwarevif.get_vif_info(self._session, [ 882.466866] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 882.466866] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] for vif in network_info: [ 882.466866] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 882.466866] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] return self._sync_wrapper(fn, *args, **kwargs) [ 882.466866] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 882.466866] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] self.wait() [ 882.466866] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 882.466866] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] self[:] = self._gt.wait() [ 882.466866] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 882.466866] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] return self._exit_event.wait() [ 882.466866] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 882.466866] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] current.throw(*self._exc) [ 882.467272] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 882.467272] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] result = function(*args, **kwargs) [ 882.467272] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 882.467272] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] return func(*args, **kwargs) [ 882.467272] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 882.467272] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] raise e [ 882.467272] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 882.467272] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] nwinfo = self.network_api.allocate_for_instance( [ 882.467272] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 882.467272] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] created_port_ids = self._update_ports_for_instance( [ 882.467272] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 882.467272] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] with excutils.save_and_reraise_exception(): [ 882.467272] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 882.467649] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] self.force_reraise() [ 882.467649] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 882.467649] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] raise self.value [ 882.467649] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 882.467649] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] updated_port = self._update_port( [ 882.467649] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 882.467649] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] _ensure_no_port_binding_failure(port) [ 882.467649] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 882.467649] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] raise exception.PortBindingFailed(port_id=port['id']) [ 882.467649] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] nova.exception.PortBindingFailed: Binding failed for port 4681981e-8791-4c18-b0e8-49b24f59baa0, please check neutron logs for more information. [ 882.467649] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] [ 882.467649] env[62499]: INFO nova.compute.manager [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Terminating instance [ 882.468822] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquiring lock "refresh_cache-0eaadb76-e2c7-4e06-b865-065ab0f014ba" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 882.592724] env[62499]: DEBUG nova.network.neutron [req-a6ce2bd3-dbac-491c-93fc-edf63ed4309e req-3121f220-94cf-4493-afb8-ad267bd99eb8 service nova] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 882.660488] env[62499]: DEBUG nova.network.neutron [req-a6ce2bd3-dbac-491c-93fc-edf63ed4309e req-3121f220-94cf-4493-afb8-ad267bd99eb8 service nova] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 882.687985] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.299s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 882.688525] env[62499]: DEBUG nova.compute.manager [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 882.691025] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.748s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 882.692633] env[62499]: INFO nova.compute.claims [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 882.810710] env[62499]: DEBUG nova.network.neutron [-] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 883.162923] env[62499]: DEBUG oslo_concurrency.lockutils [req-a6ce2bd3-dbac-491c-93fc-edf63ed4309e req-3121f220-94cf-4493-afb8-ad267bd99eb8 service nova] Releasing lock "refresh_cache-0eaadb76-e2c7-4e06-b865-065ab0f014ba" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 883.163219] env[62499]: DEBUG nova.compute.manager [req-a6ce2bd3-dbac-491c-93fc-edf63ed4309e req-3121f220-94cf-4493-afb8-ad267bd99eb8 service nova] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Received event network-vif-deleted-4681981e-8791-4c18-b0e8-49b24f59baa0 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 883.163594] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquired lock "refresh_cache-0eaadb76-e2c7-4e06-b865-065ab0f014ba" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 883.163765] env[62499]: DEBUG nova.network.neutron [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 883.198170] env[62499]: DEBUG nova.compute.utils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 883.199423] env[62499]: DEBUG nova.compute.manager [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 883.199588] env[62499]: DEBUG nova.network.neutron [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 883.245916] env[62499]: DEBUG nova.policy [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c92b9d37f796479ca3b6d9f0ed1789c8', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e444950a637b4fdab2edb1d62aac4be5', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 883.312696] env[62499]: INFO nova.compute.manager [-] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Took 1.02 seconds to deallocate network for instance. [ 883.314955] env[62499]: DEBUG nova.compute.claims [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 883.315145] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 883.494705] env[62499]: DEBUG nova.network.neutron [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Successfully created port: 9a754f05-ea5f-4418-aa84-0078099e5dce {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 883.681697] env[62499]: DEBUG nova.network.neutron [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 883.703945] env[62499]: DEBUG nova.compute.manager [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 883.781096] env[62499]: DEBUG nova.network.neutron [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 883.956440] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3dda8ab0-01b2-4e1c-8d95-5507843f1769 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 883.966287] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3a83f547-04ea-4d7f-a623-c8e153891d65 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 884.008575] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2fda7501-19ec-4bcc-95d4-873bf0688bc6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 884.016292] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19d72fae-1b6e-4b7c-af4e-861b2614e99f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 884.029750] env[62499]: DEBUG nova.compute.provider_tree [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 884.283813] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Releasing lock "refresh_cache-0eaadb76-e2c7-4e06-b865-065ab0f014ba" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 884.284258] env[62499]: DEBUG nova.compute.manager [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 884.284449] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 884.284795] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8255760d-76f9-4e0a-b47f-aba775b98614 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 884.293967] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-975be40b-a6ca-45c4-bf6e-615a782fe5c9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 884.315574] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 0eaadb76-e2c7-4e06-b865-065ab0f014ba could not be found. [ 884.315793] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 884.316057] env[62499]: INFO nova.compute.manager [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Took 0.03 seconds to destroy the instance on the hypervisor. [ 884.316256] env[62499]: DEBUG oslo.service.loopingcall [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 884.316527] env[62499]: DEBUG nova.compute.manager [-] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 884.316599] env[62499]: DEBUG nova.network.neutron [-] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 884.331266] env[62499]: DEBUG nova.network.neutron [-] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 884.405954] env[62499]: DEBUG nova.compute.manager [req-9af3af72-be8c-45a9-acc4-45dea7b063d6 req-8d6eb136-5279-4ce3-80cd-afa301955e77 service nova] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Received event network-changed-9a754f05-ea5f-4418-aa84-0078099e5dce {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 884.406317] env[62499]: DEBUG nova.compute.manager [req-9af3af72-be8c-45a9-acc4-45dea7b063d6 req-8d6eb136-5279-4ce3-80cd-afa301955e77 service nova] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Refreshing instance network info cache due to event network-changed-9a754f05-ea5f-4418-aa84-0078099e5dce. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 884.406358] env[62499]: DEBUG oslo_concurrency.lockutils [req-9af3af72-be8c-45a9-acc4-45dea7b063d6 req-8d6eb136-5279-4ce3-80cd-afa301955e77 service nova] Acquiring lock "refresh_cache-764ef9ab-22ae-45cd-8b5d-76a63346d93c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 884.406567] env[62499]: DEBUG oslo_concurrency.lockutils [req-9af3af72-be8c-45a9-acc4-45dea7b063d6 req-8d6eb136-5279-4ce3-80cd-afa301955e77 service nova] Acquired lock "refresh_cache-764ef9ab-22ae-45cd-8b5d-76a63346d93c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 884.406754] env[62499]: DEBUG nova.network.neutron [req-9af3af72-be8c-45a9-acc4-45dea7b063d6 req-8d6eb136-5279-4ce3-80cd-afa301955e77 service nova] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Refreshing network info cache for port 9a754f05-ea5f-4418-aa84-0078099e5dce {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 884.533101] env[62499]: DEBUG nova.scheduler.client.report [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 884.550702] env[62499]: ERROR nova.compute.manager [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 9a754f05-ea5f-4418-aa84-0078099e5dce, please check neutron logs for more information. [ 884.550702] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 884.550702] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 884.550702] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 884.550702] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 884.550702] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 884.550702] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 884.550702] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 884.550702] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 884.550702] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 884.550702] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 884.550702] env[62499]: ERROR nova.compute.manager raise self.value [ 884.550702] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 884.550702] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 884.550702] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 884.550702] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 884.551479] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 884.551479] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 884.551479] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 9a754f05-ea5f-4418-aa84-0078099e5dce, please check neutron logs for more information. [ 884.551479] env[62499]: ERROR nova.compute.manager [ 884.551479] env[62499]: Traceback (most recent call last): [ 884.551479] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 884.551479] env[62499]: listener.cb(fileno) [ 884.551479] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 884.551479] env[62499]: result = function(*args, **kwargs) [ 884.551479] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 884.551479] env[62499]: return func(*args, **kwargs) [ 884.551479] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 884.551479] env[62499]: raise e [ 884.551479] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 884.551479] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 884.551479] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 884.551479] env[62499]: created_port_ids = self._update_ports_for_instance( [ 884.551479] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 884.551479] env[62499]: with excutils.save_and_reraise_exception(): [ 884.551479] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 884.551479] env[62499]: self.force_reraise() [ 884.551479] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 884.551479] env[62499]: raise self.value [ 884.551479] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 884.551479] env[62499]: updated_port = self._update_port( [ 884.551479] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 884.551479] env[62499]: _ensure_no_port_binding_failure(port) [ 884.551479] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 884.551479] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 884.552628] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 9a754f05-ea5f-4418-aa84-0078099e5dce, please check neutron logs for more information. [ 884.552628] env[62499]: Removing descriptor: 16 [ 884.715783] env[62499]: DEBUG nova.compute.manager [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 884.740863] env[62499]: DEBUG nova.virt.hardware [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 884.741148] env[62499]: DEBUG nova.virt.hardware [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 884.741309] env[62499]: DEBUG nova.virt.hardware [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 884.741487] env[62499]: DEBUG nova.virt.hardware [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 884.741630] env[62499]: DEBUG nova.virt.hardware [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 884.741819] env[62499]: DEBUG nova.virt.hardware [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 884.742056] env[62499]: DEBUG nova.virt.hardware [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 884.742227] env[62499]: DEBUG nova.virt.hardware [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 884.742391] env[62499]: DEBUG nova.virt.hardware [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 884.742551] env[62499]: DEBUG nova.virt.hardware [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 884.742717] env[62499]: DEBUG nova.virt.hardware [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 884.743671] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4cfa27ec-c5e6-4e07-bca4-928e2f53f40b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 884.751770] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5064281-3d80-4932-9059-dcb4f4b1984c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 884.765275] env[62499]: ERROR nova.compute.manager [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 9a754f05-ea5f-4418-aa84-0078099e5dce, please check neutron logs for more information. [ 884.765275] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Traceback (most recent call last): [ 884.765275] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 884.765275] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] yield resources [ 884.765275] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 884.765275] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] self.driver.spawn(context, instance, image_meta, [ 884.765275] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 884.765275] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 884.765275] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 884.765275] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] vm_ref = self.build_virtual_machine(instance, [ 884.765275] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 884.765713] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] vif_infos = vmwarevif.get_vif_info(self._session, [ 884.765713] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 884.765713] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] for vif in network_info: [ 884.765713] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 884.765713] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] return self._sync_wrapper(fn, *args, **kwargs) [ 884.765713] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 884.765713] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] self.wait() [ 884.765713] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 884.765713] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] self[:] = self._gt.wait() [ 884.765713] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 884.765713] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] return self._exit_event.wait() [ 884.765713] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 884.765713] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] current.throw(*self._exc) [ 884.766129] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 884.766129] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] result = function(*args, **kwargs) [ 884.766129] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 884.766129] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] return func(*args, **kwargs) [ 884.766129] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 884.766129] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] raise e [ 884.766129] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 884.766129] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] nwinfo = self.network_api.allocate_for_instance( [ 884.766129] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 884.766129] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] created_port_ids = self._update_ports_for_instance( [ 884.766129] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 884.766129] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] with excutils.save_and_reraise_exception(): [ 884.766129] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 884.766531] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] self.force_reraise() [ 884.766531] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 884.766531] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] raise self.value [ 884.766531] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 884.766531] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] updated_port = self._update_port( [ 884.766531] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 884.766531] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] _ensure_no_port_binding_failure(port) [ 884.766531] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 884.766531] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] raise exception.PortBindingFailed(port_id=port['id']) [ 884.766531] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] nova.exception.PortBindingFailed: Binding failed for port 9a754f05-ea5f-4418-aa84-0078099e5dce, please check neutron logs for more information. [ 884.766531] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] [ 884.766531] env[62499]: INFO nova.compute.manager [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Terminating instance [ 884.769893] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "refresh_cache-764ef9ab-22ae-45cd-8b5d-76a63346d93c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 884.834486] env[62499]: DEBUG nova.network.neutron [-] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 884.923296] env[62499]: DEBUG nova.network.neutron [req-9af3af72-be8c-45a9-acc4-45dea7b063d6 req-8d6eb136-5279-4ce3-80cd-afa301955e77 service nova] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 885.006308] env[62499]: DEBUG nova.network.neutron [req-9af3af72-be8c-45a9-acc4-45dea7b063d6 req-8d6eb136-5279-4ce3-80cd-afa301955e77 service nova] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 885.037801] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.347s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 885.038334] env[62499]: DEBUG nova.compute.manager [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 885.040814] env[62499]: DEBUG oslo_concurrency.lockutils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.119s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 885.042589] env[62499]: INFO nova.compute.claims [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 885.336707] env[62499]: INFO nova.compute.manager [-] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Took 1.02 seconds to deallocate network for instance. [ 885.339027] env[62499]: DEBUG nova.compute.claims [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 885.339432] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 885.509343] env[62499]: DEBUG oslo_concurrency.lockutils [req-9af3af72-be8c-45a9-acc4-45dea7b063d6 req-8d6eb136-5279-4ce3-80cd-afa301955e77 service nova] Releasing lock "refresh_cache-764ef9ab-22ae-45cd-8b5d-76a63346d93c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 885.509769] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquired lock "refresh_cache-764ef9ab-22ae-45cd-8b5d-76a63346d93c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 885.509978] env[62499]: DEBUG nova.network.neutron [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 885.546761] env[62499]: DEBUG nova.compute.utils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 885.550522] env[62499]: DEBUG nova.compute.manager [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 885.550522] env[62499]: DEBUG nova.network.neutron [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 885.589479] env[62499]: DEBUG nova.policy [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7da3078e8bb5444c96e007cfc2d57952', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c9f95d7e954f4413b01aad519c498967', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 885.841519] env[62499]: DEBUG nova.network.neutron [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Successfully created port: 0349f93a-0a30-49e4-a1c6-0c72ee6a71c9 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 886.027508] env[62499]: DEBUG nova.network.neutron [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 886.051092] env[62499]: DEBUG nova.compute.manager [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 886.289740] env[62499]: DEBUG nova.network.neutron [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 886.320596] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3fa7211a-0d5f-4d39-b265-6929d9bf3c3a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 886.327878] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7cbb93f-be5d-4962-ae18-94b41f5a5822 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 886.356923] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff3b3220-ccff-412c-a1c1-a8fb9738e90f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 886.363606] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d69b26f0-7839-44bc-8963-a35e1b6f3d60 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 886.376240] env[62499]: DEBUG nova.compute.provider_tree [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 886.433146] env[62499]: DEBUG nova.compute.manager [req-8a155c46-83eb-4a5f-b1da-0f9f9c36a060 req-1fdc3d10-4ab0-42b2-b38f-b77b4dd85337 service nova] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Received event network-vif-deleted-9a754f05-ea5f-4418-aa84-0078099e5dce {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 886.719202] env[62499]: ERROR nova.compute.manager [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 0349f93a-0a30-49e4-a1c6-0c72ee6a71c9, please check neutron logs for more information. [ 886.719202] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 886.719202] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 886.719202] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 886.719202] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 886.719202] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 886.719202] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 886.719202] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 886.719202] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 886.719202] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 886.719202] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 886.719202] env[62499]: ERROR nova.compute.manager raise self.value [ 886.719202] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 886.719202] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 886.719202] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 886.719202] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 886.719735] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 886.719735] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 886.719735] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 0349f93a-0a30-49e4-a1c6-0c72ee6a71c9, please check neutron logs for more information. [ 886.719735] env[62499]: ERROR nova.compute.manager [ 886.719735] env[62499]: Traceback (most recent call last): [ 886.719735] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 886.719735] env[62499]: listener.cb(fileno) [ 886.719735] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 886.719735] env[62499]: result = function(*args, **kwargs) [ 886.719735] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 886.719735] env[62499]: return func(*args, **kwargs) [ 886.719735] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 886.719735] env[62499]: raise e [ 886.719735] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 886.719735] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 886.719735] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 886.719735] env[62499]: created_port_ids = self._update_ports_for_instance( [ 886.719735] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 886.719735] env[62499]: with excutils.save_and_reraise_exception(): [ 886.719735] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 886.719735] env[62499]: self.force_reraise() [ 886.719735] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 886.719735] env[62499]: raise self.value [ 886.719735] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 886.719735] env[62499]: updated_port = self._update_port( [ 886.719735] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 886.719735] env[62499]: _ensure_no_port_binding_failure(port) [ 886.719735] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 886.719735] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 886.720652] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 0349f93a-0a30-49e4-a1c6-0c72ee6a71c9, please check neutron logs for more information. [ 886.720652] env[62499]: Removing descriptor: 16 [ 886.792788] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Releasing lock "refresh_cache-764ef9ab-22ae-45cd-8b5d-76a63346d93c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 886.793239] env[62499]: DEBUG nova.compute.manager [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 886.793434] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 886.793737] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-119ffd72-4389-4e80-880a-256e3e51ccf5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 886.802835] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3acc099d-47fa-4a4f-9c44-636d0caf26ad {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 886.823260] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 764ef9ab-22ae-45cd-8b5d-76a63346d93c could not be found. [ 886.823472] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 886.823654] env[62499]: INFO nova.compute.manager [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Took 0.03 seconds to destroy the instance on the hypervisor. [ 886.823891] env[62499]: DEBUG oslo.service.loopingcall [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 886.824117] env[62499]: DEBUG nova.compute.manager [-] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 886.824214] env[62499]: DEBUG nova.network.neutron [-] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 886.840492] env[62499]: DEBUG nova.network.neutron [-] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 886.880192] env[62499]: DEBUG nova.scheduler.client.report [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 887.065027] env[62499]: DEBUG nova.compute.manager [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 887.088094] env[62499]: DEBUG nova.virt.hardware [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 887.088347] env[62499]: DEBUG nova.virt.hardware [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 887.088502] env[62499]: DEBUG nova.virt.hardware [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 887.088682] env[62499]: DEBUG nova.virt.hardware [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 887.088825] env[62499]: DEBUG nova.virt.hardware [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 887.088969] env[62499]: DEBUG nova.virt.hardware [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 887.089192] env[62499]: DEBUG nova.virt.hardware [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 887.089346] env[62499]: DEBUG nova.virt.hardware [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 887.089509] env[62499]: DEBUG nova.virt.hardware [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 887.089666] env[62499]: DEBUG nova.virt.hardware [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 887.089833] env[62499]: DEBUG nova.virt.hardware [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 887.090694] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f820fc70-e726-4db8-b484-4b3597d477cf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 887.098537] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-418041a5-dccd-4f9b-976d-de77c14ecc79 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 887.111652] env[62499]: ERROR nova.compute.manager [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 0349f93a-0a30-49e4-a1c6-0c72ee6a71c9, please check neutron logs for more information. [ 887.111652] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Traceback (most recent call last): [ 887.111652] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 887.111652] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] yield resources [ 887.111652] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 887.111652] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] self.driver.spawn(context, instance, image_meta, [ 887.111652] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 887.111652] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 887.111652] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 887.111652] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] vm_ref = self.build_virtual_machine(instance, [ 887.111652] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 887.112035] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] vif_infos = vmwarevif.get_vif_info(self._session, [ 887.112035] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 887.112035] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] for vif in network_info: [ 887.112035] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 887.112035] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] return self._sync_wrapper(fn, *args, **kwargs) [ 887.112035] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 887.112035] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] self.wait() [ 887.112035] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 887.112035] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] self[:] = self._gt.wait() [ 887.112035] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 887.112035] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] return self._exit_event.wait() [ 887.112035] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 887.112035] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] current.throw(*self._exc) [ 887.112375] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 887.112375] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] result = function(*args, **kwargs) [ 887.112375] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 887.112375] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] return func(*args, **kwargs) [ 887.112375] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 887.112375] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] raise e [ 887.112375] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 887.112375] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] nwinfo = self.network_api.allocate_for_instance( [ 887.112375] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 887.112375] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] created_port_ids = self._update_ports_for_instance( [ 887.112375] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 887.112375] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] with excutils.save_and_reraise_exception(): [ 887.112375] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 887.113023] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] self.force_reraise() [ 887.113023] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 887.113023] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] raise self.value [ 887.113023] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 887.113023] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] updated_port = self._update_port( [ 887.113023] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 887.113023] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] _ensure_no_port_binding_failure(port) [ 887.113023] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 887.113023] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] raise exception.PortBindingFailed(port_id=port['id']) [ 887.113023] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] nova.exception.PortBindingFailed: Binding failed for port 0349f93a-0a30-49e4-a1c6-0c72ee6a71c9, please check neutron logs for more information. [ 887.113023] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] [ 887.113023] env[62499]: INFO nova.compute.manager [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Terminating instance [ 887.113878] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "refresh_cache-a92509e3-d5b1-41d3-b9af-fa0af8a956c9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 887.114042] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquired lock "refresh_cache-a92509e3-d5b1-41d3-b9af-fa0af8a956c9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 887.114217] env[62499]: DEBUG nova.network.neutron [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 887.342667] env[62499]: DEBUG nova.network.neutron [-] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 887.384714] env[62499]: DEBUG oslo_concurrency.lockutils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.343s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 887.384714] env[62499]: DEBUG nova.compute.manager [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 887.387203] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.608s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 887.389017] env[62499]: INFO nova.compute.claims [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 887.633702] env[62499]: DEBUG nova.network.neutron [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 887.721505] env[62499]: DEBUG nova.network.neutron [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 887.845231] env[62499]: INFO nova.compute.manager [-] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Took 1.02 seconds to deallocate network for instance. [ 887.847548] env[62499]: DEBUG nova.compute.claims [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 887.847724] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 887.892755] env[62499]: DEBUG nova.compute.utils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 887.895918] env[62499]: DEBUG nova.compute.manager [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 887.896100] env[62499]: DEBUG nova.network.neutron [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 887.933656] env[62499]: DEBUG nova.policy [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '88c76f25140f4de28bd719de1d5ff616', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'cb9ca6d8c8ae4352bc3c7b1daa7ba288', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 888.213856] env[62499]: DEBUG nova.network.neutron [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Successfully created port: 982d43f5-d2d0-473c-a37b-1cb4ec42ab11 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 888.224454] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Releasing lock "refresh_cache-a92509e3-d5b1-41d3-b9af-fa0af8a956c9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 888.224869] env[62499]: DEBUG nova.compute.manager [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 888.225071] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 888.225360] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-403def6b-bca3-42e9-aaa5-6667456678af {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.234409] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78e15840-87eb-4a00-a644-97087dc06cc6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.254509] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a92509e3-d5b1-41d3-b9af-fa0af8a956c9 could not be found. [ 888.254720] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 888.254901] env[62499]: INFO nova.compute.manager [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Took 0.03 seconds to destroy the instance on the hypervisor. [ 888.255155] env[62499]: DEBUG oslo.service.loopingcall [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 888.255364] env[62499]: DEBUG nova.compute.manager [-] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 888.255454] env[62499]: DEBUG nova.network.neutron [-] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 888.269582] env[62499]: DEBUG nova.network.neutron [-] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 888.397366] env[62499]: DEBUG nova.compute.manager [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 888.468881] env[62499]: DEBUG nova.compute.manager [req-96e58396-456f-4183-b0e7-efbb4afec675 req-a6f6ba01-b34a-495e-ba78-7a75abec755d service nova] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Received event network-changed-0349f93a-0a30-49e4-a1c6-0c72ee6a71c9 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 888.469154] env[62499]: DEBUG nova.compute.manager [req-96e58396-456f-4183-b0e7-efbb4afec675 req-a6f6ba01-b34a-495e-ba78-7a75abec755d service nova] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Refreshing instance network info cache due to event network-changed-0349f93a-0a30-49e4-a1c6-0c72ee6a71c9. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 888.469304] env[62499]: DEBUG oslo_concurrency.lockutils [req-96e58396-456f-4183-b0e7-efbb4afec675 req-a6f6ba01-b34a-495e-ba78-7a75abec755d service nova] Acquiring lock "refresh_cache-a92509e3-d5b1-41d3-b9af-fa0af8a956c9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 888.469430] env[62499]: DEBUG oslo_concurrency.lockutils [req-96e58396-456f-4183-b0e7-efbb4afec675 req-a6f6ba01-b34a-495e-ba78-7a75abec755d service nova] Acquired lock "refresh_cache-a92509e3-d5b1-41d3-b9af-fa0af8a956c9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 888.469588] env[62499]: DEBUG nova.network.neutron [req-96e58396-456f-4183-b0e7-efbb4afec675 req-a6f6ba01-b34a-495e-ba78-7a75abec755d service nova] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Refreshing network info cache for port 0349f93a-0a30-49e4-a1c6-0c72ee6a71c9 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 888.602928] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3ba5ba7-5153-450a-b627-bfd6e23a3830 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.610846] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3331bdba-18b7-4c55-bb80-c01a9bdbed94 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.640016] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-634502ef-d11e-4803-aee0-693dd849f2f2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.647091] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-084930e2-bd04-4083-a50a-9ef9caa0a90a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.659701] env[62499]: DEBUG nova.compute.provider_tree [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 888.772296] env[62499]: DEBUG nova.network.neutron [-] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 888.990067] env[62499]: DEBUG nova.network.neutron [req-96e58396-456f-4183-b0e7-efbb4afec675 req-a6f6ba01-b34a-495e-ba78-7a75abec755d service nova] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 889.082956] env[62499]: DEBUG nova.network.neutron [req-96e58396-456f-4183-b0e7-efbb4afec675 req-a6f6ba01-b34a-495e-ba78-7a75abec755d service nova] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 889.112172] env[62499]: ERROR nova.compute.manager [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 982d43f5-d2d0-473c-a37b-1cb4ec42ab11, please check neutron logs for more information. [ 889.112172] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 889.112172] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 889.112172] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 889.112172] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 889.112172] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 889.112172] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 889.112172] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 889.112172] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 889.112172] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 889.112172] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 889.112172] env[62499]: ERROR nova.compute.manager raise self.value [ 889.112172] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 889.112172] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 889.112172] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 889.112172] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 889.112741] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 889.112741] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 889.112741] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 982d43f5-d2d0-473c-a37b-1cb4ec42ab11, please check neutron logs for more information. [ 889.112741] env[62499]: ERROR nova.compute.manager [ 889.112741] env[62499]: Traceback (most recent call last): [ 889.112741] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 889.112741] env[62499]: listener.cb(fileno) [ 889.112741] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 889.112741] env[62499]: result = function(*args, **kwargs) [ 889.112741] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 889.112741] env[62499]: return func(*args, **kwargs) [ 889.112741] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 889.112741] env[62499]: raise e [ 889.112741] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 889.112741] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 889.112741] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 889.112741] env[62499]: created_port_ids = self._update_ports_for_instance( [ 889.112741] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 889.112741] env[62499]: with excutils.save_and_reraise_exception(): [ 889.112741] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 889.112741] env[62499]: self.force_reraise() [ 889.112741] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 889.112741] env[62499]: raise self.value [ 889.112741] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 889.112741] env[62499]: updated_port = self._update_port( [ 889.112741] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 889.112741] env[62499]: _ensure_no_port_binding_failure(port) [ 889.112741] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 889.112741] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 889.113594] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 982d43f5-d2d0-473c-a37b-1cb4ec42ab11, please check neutron logs for more information. [ 889.113594] env[62499]: Removing descriptor: 16 [ 889.162775] env[62499]: DEBUG nova.scheduler.client.report [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 889.274271] env[62499]: INFO nova.compute.manager [-] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Took 1.02 seconds to deallocate network for instance. [ 889.276543] env[62499]: DEBUG nova.compute.claims [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 889.276720] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 889.410946] env[62499]: DEBUG nova.compute.manager [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 889.438308] env[62499]: DEBUG nova.virt.hardware [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 889.438552] env[62499]: DEBUG nova.virt.hardware [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 889.438713] env[62499]: DEBUG nova.virt.hardware [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 889.438877] env[62499]: DEBUG nova.virt.hardware [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 889.439033] env[62499]: DEBUG nova.virt.hardware [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 889.439186] env[62499]: DEBUG nova.virt.hardware [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 889.439390] env[62499]: DEBUG nova.virt.hardware [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 889.439576] env[62499]: DEBUG nova.virt.hardware [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 889.439779] env[62499]: DEBUG nova.virt.hardware [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 889.439964] env[62499]: DEBUG nova.virt.hardware [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 889.440177] env[62499]: DEBUG nova.virt.hardware [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 889.441049] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1da32956-0bf7-4286-a8c6-f450c9f387ee {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 889.449086] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1e655579-dad6-4c9d-9ac1-37eb9d992f0a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 889.463114] env[62499]: ERROR nova.compute.manager [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 982d43f5-d2d0-473c-a37b-1cb4ec42ab11, please check neutron logs for more information. [ 889.463114] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] Traceback (most recent call last): [ 889.463114] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 889.463114] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] yield resources [ 889.463114] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 889.463114] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] self.driver.spawn(context, instance, image_meta, [ 889.463114] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 889.463114] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 889.463114] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 889.463114] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] vm_ref = self.build_virtual_machine(instance, [ 889.463114] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 889.463522] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] vif_infos = vmwarevif.get_vif_info(self._session, [ 889.463522] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 889.463522] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] for vif in network_info: [ 889.463522] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 889.463522] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] return self._sync_wrapper(fn, *args, **kwargs) [ 889.463522] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 889.463522] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] self.wait() [ 889.463522] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 889.463522] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] self[:] = self._gt.wait() [ 889.463522] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 889.463522] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] return self._exit_event.wait() [ 889.463522] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 889.463522] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] current.throw(*self._exc) [ 889.463838] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 889.463838] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] result = function(*args, **kwargs) [ 889.463838] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 889.463838] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] return func(*args, **kwargs) [ 889.463838] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 889.463838] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] raise e [ 889.463838] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 889.463838] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] nwinfo = self.network_api.allocate_for_instance( [ 889.463838] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 889.463838] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] created_port_ids = self._update_ports_for_instance( [ 889.463838] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 889.463838] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] with excutils.save_and_reraise_exception(): [ 889.463838] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 889.464183] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] self.force_reraise() [ 889.464183] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 889.464183] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] raise self.value [ 889.464183] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 889.464183] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] updated_port = self._update_port( [ 889.464183] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 889.464183] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] _ensure_no_port_binding_failure(port) [ 889.464183] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 889.464183] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] raise exception.PortBindingFailed(port_id=port['id']) [ 889.464183] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] nova.exception.PortBindingFailed: Binding failed for port 982d43f5-d2d0-473c-a37b-1cb4ec42ab11, please check neutron logs for more information. [ 889.464183] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] [ 889.464183] env[62499]: INFO nova.compute.manager [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Terminating instance [ 889.465235] env[62499]: DEBUG oslo_concurrency.lockutils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquiring lock "refresh_cache-770160d6-4681-40e0-a812-5c491b804da4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 889.465395] env[62499]: DEBUG oslo_concurrency.lockutils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquired lock "refresh_cache-770160d6-4681-40e0-a812-5c491b804da4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 889.465560] env[62499]: DEBUG nova.network.neutron [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 889.585689] env[62499]: DEBUG oslo_concurrency.lockutils [req-96e58396-456f-4183-b0e7-efbb4afec675 req-a6f6ba01-b34a-495e-ba78-7a75abec755d service nova] Releasing lock "refresh_cache-a92509e3-d5b1-41d3-b9af-fa0af8a956c9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 889.585940] env[62499]: DEBUG nova.compute.manager [req-96e58396-456f-4183-b0e7-efbb4afec675 req-a6f6ba01-b34a-495e-ba78-7a75abec755d service nova] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Received event network-vif-deleted-0349f93a-0a30-49e4-a1c6-0c72ee6a71c9 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 889.667567] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.280s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 889.668115] env[62499]: DEBUG nova.compute.manager [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 889.670750] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.962s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 889.982555] env[62499]: DEBUG nova.network.neutron [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 890.061522] env[62499]: DEBUG nova.network.neutron [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 890.175512] env[62499]: DEBUG nova.compute.utils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 890.179816] env[62499]: DEBUG nova.compute.manager [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 890.179816] env[62499]: DEBUG nova.network.neutron [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 890.219599] env[62499]: DEBUG nova.policy [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'dc0325ff468f4160a86c0a1abb942b6f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ff1a08f012bf4871bfd8f405acc32220', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 890.345405] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2782787-522c-4fd9-ac69-c2f1982a789a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 890.352655] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8abf55f7-d3bd-4eb9-9f54-2de4f7064d52 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 890.383067] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff0d2610-c0eb-493a-bf97-f8f2bddf2036 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 890.390361] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a8385fd-9f59-44ac-b97c-1b6fc4508d71 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 890.405867] env[62499]: DEBUG nova.compute.provider_tree [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 890.490207] env[62499]: DEBUG nova.compute.manager [req-f376f6eb-ee16-4fb1-b0b3-af2dee2bef20 req-9a1d6a58-b51a-470d-b6b0-785764816e92 service nova] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Received event network-changed-982d43f5-d2d0-473c-a37b-1cb4ec42ab11 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 890.490459] env[62499]: DEBUG nova.compute.manager [req-f376f6eb-ee16-4fb1-b0b3-af2dee2bef20 req-9a1d6a58-b51a-470d-b6b0-785764816e92 service nova] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Refreshing instance network info cache due to event network-changed-982d43f5-d2d0-473c-a37b-1cb4ec42ab11. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 890.490641] env[62499]: DEBUG oslo_concurrency.lockutils [req-f376f6eb-ee16-4fb1-b0b3-af2dee2bef20 req-9a1d6a58-b51a-470d-b6b0-785764816e92 service nova] Acquiring lock "refresh_cache-770160d6-4681-40e0-a812-5c491b804da4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 890.504175] env[62499]: DEBUG nova.network.neutron [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Successfully created port: 765be7c7-1b25-4959-b478-d5f415357706 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 890.564819] env[62499]: DEBUG oslo_concurrency.lockutils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Releasing lock "refresh_cache-770160d6-4681-40e0-a812-5c491b804da4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 890.565242] env[62499]: DEBUG nova.compute.manager [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 890.565434] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 890.565723] env[62499]: DEBUG oslo_concurrency.lockutils [req-f376f6eb-ee16-4fb1-b0b3-af2dee2bef20 req-9a1d6a58-b51a-470d-b6b0-785764816e92 service nova] Acquired lock "refresh_cache-770160d6-4681-40e0-a812-5c491b804da4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 890.565916] env[62499]: DEBUG nova.network.neutron [req-f376f6eb-ee16-4fb1-b0b3-af2dee2bef20 req-9a1d6a58-b51a-470d-b6b0-785764816e92 service nova] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Refreshing network info cache for port 982d43f5-d2d0-473c-a37b-1cb4ec42ab11 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 890.566866] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-12d739d8-36f8-4f3c-ab21-6afe7a0d5a08 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 890.579232] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80d258b6-bdc1-491a-8314-99010fd8314c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 890.600173] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 770160d6-4681-40e0-a812-5c491b804da4 could not be found. [ 890.600452] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 890.600646] env[62499]: INFO nova.compute.manager [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Took 0.04 seconds to destroy the instance on the hypervisor. [ 890.601136] env[62499]: DEBUG oslo.service.loopingcall [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 890.601265] env[62499]: DEBUG nova.compute.manager [-] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 890.601311] env[62499]: DEBUG nova.network.neutron [-] [instance: 770160d6-4681-40e0-a812-5c491b804da4] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 890.616957] env[62499]: DEBUG nova.network.neutron [-] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 890.682447] env[62499]: DEBUG nova.compute.manager [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 890.910997] env[62499]: DEBUG nova.scheduler.client.report [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 891.089608] env[62499]: DEBUG nova.network.neutron [req-f376f6eb-ee16-4fb1-b0b3-af2dee2bef20 req-9a1d6a58-b51a-470d-b6b0-785764816e92 service nova] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 891.119132] env[62499]: DEBUG nova.network.neutron [-] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 891.180160] env[62499]: DEBUG nova.network.neutron [req-f376f6eb-ee16-4fb1-b0b3-af2dee2bef20 req-9a1d6a58-b51a-470d-b6b0-785764816e92 service nova] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 891.351579] env[62499]: ERROR nova.compute.manager [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 765be7c7-1b25-4959-b478-d5f415357706, please check neutron logs for more information. [ 891.351579] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 891.351579] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 891.351579] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 891.351579] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 891.351579] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 891.351579] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 891.351579] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 891.351579] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 891.351579] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 891.351579] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 891.351579] env[62499]: ERROR nova.compute.manager raise self.value [ 891.351579] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 891.351579] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 891.351579] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 891.351579] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 891.352112] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 891.352112] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 891.352112] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 765be7c7-1b25-4959-b478-d5f415357706, please check neutron logs for more information. [ 891.352112] env[62499]: ERROR nova.compute.manager [ 891.352112] env[62499]: Traceback (most recent call last): [ 891.352112] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 891.352112] env[62499]: listener.cb(fileno) [ 891.352112] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 891.352112] env[62499]: result = function(*args, **kwargs) [ 891.352112] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 891.352112] env[62499]: return func(*args, **kwargs) [ 891.352112] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 891.352112] env[62499]: raise e [ 891.352112] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 891.352112] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 891.352112] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 891.352112] env[62499]: created_port_ids = self._update_ports_for_instance( [ 891.352112] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 891.352112] env[62499]: with excutils.save_and_reraise_exception(): [ 891.352112] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 891.352112] env[62499]: self.force_reraise() [ 891.352112] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 891.352112] env[62499]: raise self.value [ 891.352112] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 891.352112] env[62499]: updated_port = self._update_port( [ 891.352112] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 891.352112] env[62499]: _ensure_no_port_binding_failure(port) [ 891.352112] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 891.352112] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 891.352888] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 765be7c7-1b25-4959-b478-d5f415357706, please check neutron logs for more information. [ 891.352888] env[62499]: Removing descriptor: 16 [ 891.416998] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.745s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 891.416998] env[62499]: ERROR nova.compute.manager [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 296e37b0-c4ab-4591-9b5f-d51c80bb0f9c, please check neutron logs for more information. [ 891.416998] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Traceback (most recent call last): [ 891.416998] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 891.416998] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] self.driver.spawn(context, instance, image_meta, [ 891.416998] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 891.416998] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] self._vmops.spawn(context, instance, image_meta, injected_files, [ 891.416998] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 891.416998] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] vm_ref = self.build_virtual_machine(instance, [ 891.417331] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 891.417331] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] vif_infos = vmwarevif.get_vif_info(self._session, [ 891.417331] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 891.417331] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] for vif in network_info: [ 891.417331] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 891.417331] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] return self._sync_wrapper(fn, *args, **kwargs) [ 891.417331] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 891.417331] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] self.wait() [ 891.417331] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 891.417331] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] self[:] = self._gt.wait() [ 891.417331] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 891.417331] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] return self._exit_event.wait() [ 891.417331] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 891.417655] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] current.throw(*self._exc) [ 891.417655] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 891.417655] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] result = function(*args, **kwargs) [ 891.417655] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 891.417655] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] return func(*args, **kwargs) [ 891.417655] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 891.417655] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] raise e [ 891.417655] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 891.417655] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] nwinfo = self.network_api.allocate_for_instance( [ 891.417655] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 891.417655] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] created_port_ids = self._update_ports_for_instance( [ 891.417655] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 891.417655] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] with excutils.save_and_reraise_exception(): [ 891.418163] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 891.418163] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] self.force_reraise() [ 891.418163] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 891.418163] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] raise self.value [ 891.418163] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 891.418163] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] updated_port = self._update_port( [ 891.418163] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 891.418163] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] _ensure_no_port_binding_failure(port) [ 891.418163] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 891.418163] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] raise exception.PortBindingFailed(port_id=port['id']) [ 891.418163] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] nova.exception.PortBindingFailed: Binding failed for port 296e37b0-c4ab-4591-9b5f-d51c80bb0f9c, please check neutron logs for more information. [ 891.418163] env[62499]: ERROR nova.compute.manager [instance: c1961f60-06d8-44d8-999f-d498c3667adc] [ 891.418484] env[62499]: DEBUG nova.compute.utils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Binding failed for port 296e37b0-c4ab-4591-9b5f-d51c80bb0f9c, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 891.418882] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 17.426s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 891.421028] env[62499]: DEBUG nova.compute.manager [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Build of instance c1961f60-06d8-44d8-999f-d498c3667adc was re-scheduled: Binding failed for port 296e37b0-c4ab-4591-9b5f-d51c80bb0f9c, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 891.421406] env[62499]: DEBUG nova.compute.manager [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 891.421893] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquiring lock "refresh_cache-c1961f60-06d8-44d8-999f-d498c3667adc" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 891.421893] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquired lock "refresh_cache-c1961f60-06d8-44d8-999f-d498c3667adc" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 891.422011] env[62499]: DEBUG nova.network.neutron [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 891.621627] env[62499]: INFO nova.compute.manager [-] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Took 1.02 seconds to deallocate network for instance. [ 891.623804] env[62499]: DEBUG nova.compute.claims [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 891.624088] env[62499]: DEBUG oslo_concurrency.lockutils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 891.681811] env[62499]: DEBUG oslo_concurrency.lockutils [req-f376f6eb-ee16-4fb1-b0b3-af2dee2bef20 req-9a1d6a58-b51a-470d-b6b0-785764816e92 service nova] Releasing lock "refresh_cache-770160d6-4681-40e0-a812-5c491b804da4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 891.682080] env[62499]: DEBUG nova.compute.manager [req-f376f6eb-ee16-4fb1-b0b3-af2dee2bef20 req-9a1d6a58-b51a-470d-b6b0-785764816e92 service nova] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Received event network-vif-deleted-982d43f5-d2d0-473c-a37b-1cb4ec42ab11 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 891.691402] env[62499]: DEBUG nova.compute.manager [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 891.714374] env[62499]: DEBUG nova.virt.hardware [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 891.714609] env[62499]: DEBUG nova.virt.hardware [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 891.714765] env[62499]: DEBUG nova.virt.hardware [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 891.714944] env[62499]: DEBUG nova.virt.hardware [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 891.715104] env[62499]: DEBUG nova.virt.hardware [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 891.715250] env[62499]: DEBUG nova.virt.hardware [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 891.715454] env[62499]: DEBUG nova.virt.hardware [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 891.715608] env[62499]: DEBUG nova.virt.hardware [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 891.715766] env[62499]: DEBUG nova.virt.hardware [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 891.715922] env[62499]: DEBUG nova.virt.hardware [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 891.716105] env[62499]: DEBUG nova.virt.hardware [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 891.716938] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b018de65-a3ce-4526-9305-7615c424056f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.725124] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e1f27723-ea5b-48d2-b85b-e65815b62f9e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.739322] env[62499]: ERROR nova.compute.manager [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 765be7c7-1b25-4959-b478-d5f415357706, please check neutron logs for more information. [ 891.739322] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Traceback (most recent call last): [ 891.739322] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 891.739322] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] yield resources [ 891.739322] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 891.739322] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] self.driver.spawn(context, instance, image_meta, [ 891.739322] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 891.739322] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 891.739322] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 891.739322] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] vm_ref = self.build_virtual_machine(instance, [ 891.739322] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 891.739681] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] vif_infos = vmwarevif.get_vif_info(self._session, [ 891.739681] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 891.739681] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] for vif in network_info: [ 891.739681] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 891.739681] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] return self._sync_wrapper(fn, *args, **kwargs) [ 891.739681] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 891.739681] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] self.wait() [ 891.739681] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 891.739681] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] self[:] = self._gt.wait() [ 891.739681] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 891.739681] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] return self._exit_event.wait() [ 891.739681] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 891.739681] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] current.throw(*self._exc) [ 891.740153] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 891.740153] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] result = function(*args, **kwargs) [ 891.740153] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 891.740153] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] return func(*args, **kwargs) [ 891.740153] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 891.740153] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] raise e [ 891.740153] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 891.740153] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] nwinfo = self.network_api.allocate_for_instance( [ 891.740153] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 891.740153] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] created_port_ids = self._update_ports_for_instance( [ 891.740153] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 891.740153] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] with excutils.save_and_reraise_exception(): [ 891.740153] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 891.740546] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] self.force_reraise() [ 891.740546] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 891.740546] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] raise self.value [ 891.740546] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 891.740546] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] updated_port = self._update_port( [ 891.740546] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 891.740546] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] _ensure_no_port_binding_failure(port) [ 891.740546] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 891.740546] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] raise exception.PortBindingFailed(port_id=port['id']) [ 891.740546] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] nova.exception.PortBindingFailed: Binding failed for port 765be7c7-1b25-4959-b478-d5f415357706, please check neutron logs for more information. [ 891.740546] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] [ 891.740546] env[62499]: INFO nova.compute.manager [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Terminating instance [ 891.741534] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Acquiring lock "refresh_cache-cbe51201-c036-4f90-9372-8cc2d033f7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 891.741696] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Acquired lock "refresh_cache-cbe51201-c036-4f90-9372-8cc2d033f7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 891.741860] env[62499]: DEBUG nova.network.neutron [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 891.943365] env[62499]: DEBUG nova.network.neutron [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 892.024024] env[62499]: DEBUG nova.network.neutron [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 892.267418] env[62499]: DEBUG nova.network.neutron [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 892.343703] env[62499]: DEBUG nova.network.neutron [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 892.516936] env[62499]: DEBUG nova.compute.manager [req-3ec1f57e-d611-4777-b259-639a04e307c2 req-18843a19-e45c-4ad6-9422-b32537ca0534 service nova] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Received event network-changed-765be7c7-1b25-4959-b478-d5f415357706 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 892.517292] env[62499]: DEBUG nova.compute.manager [req-3ec1f57e-d611-4777-b259-639a04e307c2 req-18843a19-e45c-4ad6-9422-b32537ca0534 service nova] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Refreshing instance network info cache due to event network-changed-765be7c7-1b25-4959-b478-d5f415357706. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 892.517607] env[62499]: DEBUG oslo_concurrency.lockutils [req-3ec1f57e-d611-4777-b259-639a04e307c2 req-18843a19-e45c-4ad6-9422-b32537ca0534 service nova] Acquiring lock "refresh_cache-cbe51201-c036-4f90-9372-8cc2d033f7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 892.526129] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Releasing lock "refresh_cache-c1961f60-06d8-44d8-999f-d498c3667adc" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 892.526427] env[62499]: DEBUG nova.compute.manager [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 892.526708] env[62499]: DEBUG nova.compute.manager [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 892.526976] env[62499]: DEBUG nova.network.neutron [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 892.543063] env[62499]: DEBUG nova.network.neutron [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 892.846834] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Releasing lock "refresh_cache-cbe51201-c036-4f90-9372-8cc2d033f7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 892.847324] env[62499]: DEBUG nova.compute.manager [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 892.847520] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 892.847916] env[62499]: DEBUG oslo_concurrency.lockutils [req-3ec1f57e-d611-4777-b259-639a04e307c2 req-18843a19-e45c-4ad6-9422-b32537ca0534 service nova] Acquired lock "refresh_cache-cbe51201-c036-4f90-9372-8cc2d033f7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 892.848056] env[62499]: DEBUG nova.network.neutron [req-3ec1f57e-d611-4777-b259-639a04e307c2 req-18843a19-e45c-4ad6-9422-b32537ca0534 service nova] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Refreshing network info cache for port 765be7c7-1b25-4959-b478-d5f415357706 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 892.849158] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-aa466471-c63c-4d81-8d04-bc49543648a8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 892.859285] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ee7564b-8835-427f-a0d2-8fa7d0639b63 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 892.880481] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance cbe51201-c036-4f90-9372-8cc2d033f7a2 could not be found. [ 892.880718] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 892.880874] env[62499]: INFO nova.compute.manager [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Took 0.03 seconds to destroy the instance on the hypervisor. [ 892.881176] env[62499]: DEBUG oslo.service.loopingcall [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 892.881402] env[62499]: DEBUG nova.compute.manager [-] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 892.881494] env[62499]: DEBUG nova.network.neutron [-] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 892.897106] env[62499]: DEBUG nova.network.neutron [-] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 892.950345] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance c1961f60-06d8-44d8-999f-d498c3667adc has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 892.950506] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance ba8fc3e0-dae8-4910-858b-dab1de72089e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 892.950629] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 0eaadb76-e2c7-4e06-b865-065ab0f014ba actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 256, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 892.950746] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 764ef9ab-22ae-45cd-8b5d-76a63346d93c actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 892.950860] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance a92509e3-d5b1-41d3-b9af-fa0af8a956c9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 892.950980] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 770160d6-4681-40e0-a812-5c491b804da4 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 892.951106] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance cbe51201-c036-4f90-9372-8cc2d033f7a2 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 893.045532] env[62499]: DEBUG nova.network.neutron [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 893.366559] env[62499]: DEBUG nova.network.neutron [req-3ec1f57e-d611-4777-b259-639a04e307c2 req-18843a19-e45c-4ad6-9422-b32537ca0534 service nova] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 893.400189] env[62499]: DEBUG nova.network.neutron [-] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 893.436108] env[62499]: DEBUG nova.network.neutron [req-3ec1f57e-d611-4777-b259-639a04e307c2 req-18843a19-e45c-4ad6-9422-b32537ca0534 service nova] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 893.456044] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance a10b9328-394e-4673-928e-e6fa87b8c844 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 893.547754] env[62499]: INFO nova.compute.manager [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: c1961f60-06d8-44d8-999f-d498c3667adc] Took 1.02 seconds to deallocate network for instance. [ 893.904417] env[62499]: INFO nova.compute.manager [-] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Took 1.02 seconds to deallocate network for instance. [ 893.906774] env[62499]: DEBUG nova.compute.claims [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 893.906958] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 893.938612] env[62499]: DEBUG oslo_concurrency.lockutils [req-3ec1f57e-d611-4777-b259-639a04e307c2 req-18843a19-e45c-4ad6-9422-b32537ca0534 service nova] Releasing lock "refresh_cache-cbe51201-c036-4f90-9372-8cc2d033f7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 893.938848] env[62499]: DEBUG nova.compute.manager [req-3ec1f57e-d611-4777-b259-639a04e307c2 req-18843a19-e45c-4ad6-9422-b32537ca0534 service nova] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Received event network-vif-deleted-765be7c7-1b25-4959-b478-d5f415357706 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 893.959221] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance d308417b-47d0-423a-a603-56d9251ef818 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 894.465073] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance b534fcdb-b0db-4e07-85fc-696dc58168aa has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 894.576854] env[62499]: INFO nova.scheduler.client.report [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Deleted allocations for instance c1961f60-06d8-44d8-999f-d498c3667adc [ 894.968061] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance f1b5512b-702f-45b1-9844-13a1218c31b3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 895.084855] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d67bf495-4ce4-44a5-8cd3-fd105e76c45d tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "c1961f60-06d8-44d8-999f-d498c3667adc" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 147.156s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 895.470674] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 4af6c7f4-b774-4c3f-80af-e8704a904c38 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 895.587513] env[62499]: DEBUG nova.compute.manager [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 895.974109] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 4fa63411-a721-4e34-a08d-90587b0b34d3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 896.108451] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 896.477871] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 88466440-7392-43d3-83ec-45229516a3ef has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 896.981636] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 896.981972] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Total usable vcpus: 48, total allocated vcpus: 6 {{(pid=62499) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 896.982232] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1728MB phys_disk=200GB used_disk=6GB total_vcpus=48 used_vcpus=6 pci_stats=[] {{(pid=62499) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 897.135896] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5867fc6b-f753-487b-b584-9e6835fadd85 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.144564] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-835dcb28-deb7-49b6-9554-ac5d3a13f8c1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.172748] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f14a16a4-dbc6-4b2f-ae7d-47f793bd9279 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.179643] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9b426e42-ad59-496e-92a0-4dbcffacfae0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.194051] env[62499]: DEBUG nova.compute.provider_tree [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 897.697072] env[62499]: DEBUG nova.scheduler.client.report [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 898.201454] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62499) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 898.201703] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 6.783s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 898.202010] env[62499]: DEBUG oslo_concurrency.lockutils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 23.129s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 898.203551] env[62499]: INFO nova.compute.claims [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 899.360230] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb0106f2-f30c-4443-b595-c8e49d255933 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.368810] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e58413c-0d12-49c5-9af1-846901d16a83 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.397212] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa2c920b-5146-4ced-bfb7-b2c50878d604 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.404069] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7c4931a-f873-4567-9d11-69eb38a8325a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.416474] env[62499]: DEBUG nova.compute.provider_tree [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 899.919242] env[62499]: DEBUG nova.scheduler.client.report [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 900.425188] env[62499]: DEBUG oslo_concurrency.lockutils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.223s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 900.425767] env[62499]: DEBUG nova.compute.manager [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 900.428284] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 23.575s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 900.429653] env[62499]: INFO nova.compute.claims [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 900.934513] env[62499]: DEBUG nova.compute.utils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 900.937727] env[62499]: DEBUG nova.compute.manager [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 900.937922] env[62499]: DEBUG nova.network.neutron [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 900.994028] env[62499]: DEBUG nova.policy [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f4f4a5ed581245e1a4fd36af472962ef', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c0f234b7555a473ebc3191c912d57c40', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 901.249396] env[62499]: DEBUG nova.network.neutron [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Successfully created port: 91890880-66f0-40a4-a58f-6ad5b358a57a {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 901.438409] env[62499]: DEBUG nova.compute.manager [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 901.617404] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c42a073-89fa-4ff1-a0a3-122c6cecf20f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.625132] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7f1b05d-1644-46c7-a672-16cdeff1960e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.654133] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e182c723-5bd6-43d3-a791-492b8e665b04 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.661346] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6fdccbb2-7442-4065-8e78-1cb2a3387dfd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.674048] env[62499]: DEBUG nova.compute.provider_tree [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 902.078846] env[62499]: DEBUG nova.compute.manager [req-fe53d34e-a070-415a-953b-0f900aac02c8 req-33e1063b-ade6-4959-9eb0-32d756047a24 service nova] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Received event network-changed-91890880-66f0-40a4-a58f-6ad5b358a57a {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 902.079026] env[62499]: DEBUG nova.compute.manager [req-fe53d34e-a070-415a-953b-0f900aac02c8 req-33e1063b-ade6-4959-9eb0-32d756047a24 service nova] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Refreshing instance network info cache due to event network-changed-91890880-66f0-40a4-a58f-6ad5b358a57a. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 902.079407] env[62499]: DEBUG oslo_concurrency.lockutils [req-fe53d34e-a070-415a-953b-0f900aac02c8 req-33e1063b-ade6-4959-9eb0-32d756047a24 service nova] Acquiring lock "refresh_cache-a10b9328-394e-4673-928e-e6fa87b8c844" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 902.079603] env[62499]: DEBUG oslo_concurrency.lockutils [req-fe53d34e-a070-415a-953b-0f900aac02c8 req-33e1063b-ade6-4959-9eb0-32d756047a24 service nova] Acquired lock "refresh_cache-a10b9328-394e-4673-928e-e6fa87b8c844" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 902.079714] env[62499]: DEBUG nova.network.neutron [req-fe53d34e-a070-415a-953b-0f900aac02c8 req-33e1063b-ade6-4959-9eb0-32d756047a24 service nova] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Refreshing network info cache for port 91890880-66f0-40a4-a58f-6ad5b358a57a {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 902.177397] env[62499]: DEBUG nova.scheduler.client.report [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 902.226579] env[62499]: ERROR nova.compute.manager [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 91890880-66f0-40a4-a58f-6ad5b358a57a, please check neutron logs for more information. [ 902.226579] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 902.226579] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 902.226579] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 902.226579] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 902.226579] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 902.226579] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 902.226579] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 902.226579] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 902.226579] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 902.226579] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 902.226579] env[62499]: ERROR nova.compute.manager raise self.value [ 902.226579] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 902.226579] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 902.226579] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 902.226579] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 902.227269] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 902.227269] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 902.227269] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 91890880-66f0-40a4-a58f-6ad5b358a57a, please check neutron logs for more information. [ 902.227269] env[62499]: ERROR nova.compute.manager [ 902.227269] env[62499]: Traceback (most recent call last): [ 902.227269] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 902.227269] env[62499]: listener.cb(fileno) [ 902.227269] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 902.227269] env[62499]: result = function(*args, **kwargs) [ 902.227269] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 902.227269] env[62499]: return func(*args, **kwargs) [ 902.227269] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 902.227269] env[62499]: raise e [ 902.227269] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 902.227269] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 902.227269] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 902.227269] env[62499]: created_port_ids = self._update_ports_for_instance( [ 902.227269] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 902.227269] env[62499]: with excutils.save_and_reraise_exception(): [ 902.227269] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 902.227269] env[62499]: self.force_reraise() [ 902.227269] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 902.227269] env[62499]: raise self.value [ 902.227269] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 902.227269] env[62499]: updated_port = self._update_port( [ 902.227269] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 902.227269] env[62499]: _ensure_no_port_binding_failure(port) [ 902.227269] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 902.227269] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 902.228111] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 91890880-66f0-40a4-a58f-6ad5b358a57a, please check neutron logs for more information. [ 902.228111] env[62499]: Removing descriptor: 16 [ 902.450546] env[62499]: DEBUG nova.compute.manager [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 902.474553] env[62499]: DEBUG nova.virt.hardware [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 902.474832] env[62499]: DEBUG nova.virt.hardware [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 902.475011] env[62499]: DEBUG nova.virt.hardware [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 902.475211] env[62499]: DEBUG nova.virt.hardware [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 902.475358] env[62499]: DEBUG nova.virt.hardware [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 902.475505] env[62499]: DEBUG nova.virt.hardware [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 902.475706] env[62499]: DEBUG nova.virt.hardware [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 902.475863] env[62499]: DEBUG nova.virt.hardware [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 902.476038] env[62499]: DEBUG nova.virt.hardware [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 902.476207] env[62499]: DEBUG nova.virt.hardware [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 902.476477] env[62499]: DEBUG nova.virt.hardware [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 902.477304] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-68664b7d-55ae-48bf-9771-5eb40bba1ac2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 902.485270] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-619aa902-8b3b-4fd5-8b2b-d435c9f1248a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 902.499027] env[62499]: ERROR nova.compute.manager [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 91890880-66f0-40a4-a58f-6ad5b358a57a, please check neutron logs for more information. [ 902.499027] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Traceback (most recent call last): [ 902.499027] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 902.499027] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] yield resources [ 902.499027] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 902.499027] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] self.driver.spawn(context, instance, image_meta, [ 902.499027] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 902.499027] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] self._vmops.spawn(context, instance, image_meta, injected_files, [ 902.499027] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 902.499027] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] vm_ref = self.build_virtual_machine(instance, [ 902.499027] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 902.499474] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] vif_infos = vmwarevif.get_vif_info(self._session, [ 902.499474] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 902.499474] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] for vif in network_info: [ 902.499474] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 902.499474] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] return self._sync_wrapper(fn, *args, **kwargs) [ 902.499474] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 902.499474] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] self.wait() [ 902.499474] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 902.499474] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] self[:] = self._gt.wait() [ 902.499474] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 902.499474] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] return self._exit_event.wait() [ 902.499474] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 902.499474] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] current.throw(*self._exc) [ 902.499858] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 902.499858] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] result = function(*args, **kwargs) [ 902.499858] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 902.499858] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] return func(*args, **kwargs) [ 902.499858] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 902.499858] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] raise e [ 902.499858] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 902.499858] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] nwinfo = self.network_api.allocate_for_instance( [ 902.499858] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 902.499858] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] created_port_ids = self._update_ports_for_instance( [ 902.499858] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 902.499858] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] with excutils.save_and_reraise_exception(): [ 902.499858] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 902.500303] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] self.force_reraise() [ 902.500303] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 902.500303] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] raise self.value [ 902.500303] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 902.500303] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] updated_port = self._update_port( [ 902.500303] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 902.500303] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] _ensure_no_port_binding_failure(port) [ 902.500303] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 902.500303] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] raise exception.PortBindingFailed(port_id=port['id']) [ 902.500303] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] nova.exception.PortBindingFailed: Binding failed for port 91890880-66f0-40a4-a58f-6ad5b358a57a, please check neutron logs for more information. [ 902.500303] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] [ 902.500303] env[62499]: INFO nova.compute.manager [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Terminating instance [ 902.501641] env[62499]: DEBUG oslo_concurrency.lockutils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Acquiring lock "refresh_cache-a10b9328-394e-4673-928e-e6fa87b8c844" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 902.596918] env[62499]: DEBUG nova.network.neutron [req-fe53d34e-a070-415a-953b-0f900aac02c8 req-33e1063b-ade6-4959-9eb0-32d756047a24 service nova] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 902.664624] env[62499]: DEBUG nova.network.neutron [req-fe53d34e-a070-415a-953b-0f900aac02c8 req-33e1063b-ade6-4959-9eb0-32d756047a24 service nova] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 902.681509] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.253s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 902.681967] env[62499]: DEBUG nova.compute.manager [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 902.684264] env[62499]: DEBUG oslo_concurrency.lockutils [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 23.972s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 902.685596] env[62499]: INFO nova.compute.claims [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 903.167603] env[62499]: DEBUG oslo_concurrency.lockutils [req-fe53d34e-a070-415a-953b-0f900aac02c8 req-33e1063b-ade6-4959-9eb0-32d756047a24 service nova] Releasing lock "refresh_cache-a10b9328-394e-4673-928e-e6fa87b8c844" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 903.168073] env[62499]: DEBUG oslo_concurrency.lockutils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Acquired lock "refresh_cache-a10b9328-394e-4673-928e-e6fa87b8c844" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 903.168263] env[62499]: DEBUG nova.network.neutron [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 903.189613] env[62499]: DEBUG nova.compute.utils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 903.192617] env[62499]: DEBUG nova.compute.manager [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 903.192617] env[62499]: DEBUG nova.network.neutron [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 903.236270] env[62499]: DEBUG nova.policy [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2d9df254dd064811923728d85324d67d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0accdd542a294475857a3c8b139b536f', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 903.496638] env[62499]: DEBUG nova.network.neutron [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Successfully created port: aa3bd8c0-68d7-4a18-86d7-24b006731bea {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 903.686982] env[62499]: DEBUG nova.network.neutron [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 903.693546] env[62499]: DEBUG nova.compute.manager [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 903.770660] env[62499]: DEBUG nova.network.neutron [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 903.907110] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-772f9708-ba01-447d-8dc7-4f505fdb5a4f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.918452] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-681964dd-031d-4aea-8bed-91fb98b493a9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.971228] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3a5e97cd-90d8-4e61-b754-10613800882f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.982504] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3db8aa2b-a582-4b85-89d9-3f0f68686f3d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 904.003585] env[62499]: DEBUG nova.compute.provider_tree [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 904.105484] env[62499]: DEBUG nova.compute.manager [req-5528c3c6-0d55-402d-a627-fe3251ca7bac req-007937e0-b6a3-4536-b238-40ce325dfa55 service nova] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Received event network-vif-deleted-91890880-66f0-40a4-a58f-6ad5b358a57a {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 904.272977] env[62499]: DEBUG oslo_concurrency.lockutils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Releasing lock "refresh_cache-a10b9328-394e-4673-928e-e6fa87b8c844" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 904.273390] env[62499]: DEBUG nova.compute.manager [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 904.273576] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 904.273858] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ec14ecb4-522d-402e-b970-2dfe277ea314 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 904.282914] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e9655729-d345-4420-8f90-463d625c6a4e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 904.304607] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a10b9328-394e-4673-928e-e6fa87b8c844 could not be found. [ 904.304819] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 904.304994] env[62499]: INFO nova.compute.manager [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Took 0.03 seconds to destroy the instance on the hypervisor. [ 904.305249] env[62499]: DEBUG oslo.service.loopingcall [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 904.305463] env[62499]: DEBUG nova.compute.manager [-] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 904.305558] env[62499]: DEBUG nova.network.neutron [-] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 904.319665] env[62499]: DEBUG nova.network.neutron [-] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 904.410083] env[62499]: ERROR nova.compute.manager [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port aa3bd8c0-68d7-4a18-86d7-24b006731bea, please check neutron logs for more information. [ 904.410083] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 904.410083] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 904.410083] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 904.410083] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 904.410083] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 904.410083] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 904.410083] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 904.410083] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 904.410083] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 904.410083] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 904.410083] env[62499]: ERROR nova.compute.manager raise self.value [ 904.410083] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 904.410083] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 904.410083] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 904.410083] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 904.410586] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 904.410586] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 904.410586] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port aa3bd8c0-68d7-4a18-86d7-24b006731bea, please check neutron logs for more information. [ 904.410586] env[62499]: ERROR nova.compute.manager [ 904.410586] env[62499]: Traceback (most recent call last): [ 904.410586] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 904.410586] env[62499]: listener.cb(fileno) [ 904.410586] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 904.410586] env[62499]: result = function(*args, **kwargs) [ 904.410586] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 904.410586] env[62499]: return func(*args, **kwargs) [ 904.410586] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 904.410586] env[62499]: raise e [ 904.410586] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 904.410586] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 904.410586] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 904.410586] env[62499]: created_port_ids = self._update_ports_for_instance( [ 904.410586] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 904.410586] env[62499]: with excutils.save_and_reraise_exception(): [ 904.410586] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 904.410586] env[62499]: self.force_reraise() [ 904.410586] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 904.410586] env[62499]: raise self.value [ 904.410586] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 904.410586] env[62499]: updated_port = self._update_port( [ 904.410586] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 904.410586] env[62499]: _ensure_no_port_binding_failure(port) [ 904.410586] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 904.410586] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 904.411433] env[62499]: nova.exception.PortBindingFailed: Binding failed for port aa3bd8c0-68d7-4a18-86d7-24b006731bea, please check neutron logs for more information. [ 904.411433] env[62499]: Removing descriptor: 16 [ 904.509087] env[62499]: DEBUG nova.scheduler.client.report [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 904.709616] env[62499]: DEBUG nova.compute.manager [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 904.733888] env[62499]: DEBUG nova.virt.hardware [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 904.734187] env[62499]: DEBUG nova.virt.hardware [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 904.734352] env[62499]: DEBUG nova.virt.hardware [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 904.734532] env[62499]: DEBUG nova.virt.hardware [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 904.734678] env[62499]: DEBUG nova.virt.hardware [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 904.734823] env[62499]: DEBUG nova.virt.hardware [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 904.735035] env[62499]: DEBUG nova.virt.hardware [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 904.735202] env[62499]: DEBUG nova.virt.hardware [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 904.735366] env[62499]: DEBUG nova.virt.hardware [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 904.735525] env[62499]: DEBUG nova.virt.hardware [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 904.735692] env[62499]: DEBUG nova.virt.hardware [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 904.736544] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8911a4e-29f0-45ca-aad3-aaf4bb41a3f3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 904.744330] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a6b7502-fd67-4228-8b17-e90781dd338d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 904.757691] env[62499]: ERROR nova.compute.manager [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port aa3bd8c0-68d7-4a18-86d7-24b006731bea, please check neutron logs for more information. [ 904.757691] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] Traceback (most recent call last): [ 904.757691] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 904.757691] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] yield resources [ 904.757691] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 904.757691] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] self.driver.spawn(context, instance, image_meta, [ 904.757691] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 904.757691] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] self._vmops.spawn(context, instance, image_meta, injected_files, [ 904.757691] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 904.757691] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] vm_ref = self.build_virtual_machine(instance, [ 904.757691] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 904.758164] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] vif_infos = vmwarevif.get_vif_info(self._session, [ 904.758164] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 904.758164] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] for vif in network_info: [ 904.758164] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 904.758164] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] return self._sync_wrapper(fn, *args, **kwargs) [ 904.758164] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 904.758164] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] self.wait() [ 904.758164] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 904.758164] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] self[:] = self._gt.wait() [ 904.758164] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 904.758164] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] return self._exit_event.wait() [ 904.758164] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 904.758164] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] current.throw(*self._exc) [ 904.758537] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 904.758537] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] result = function(*args, **kwargs) [ 904.758537] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 904.758537] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] return func(*args, **kwargs) [ 904.758537] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 904.758537] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] raise e [ 904.758537] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 904.758537] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] nwinfo = self.network_api.allocate_for_instance( [ 904.758537] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 904.758537] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] created_port_ids = self._update_ports_for_instance( [ 904.758537] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 904.758537] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] with excutils.save_and_reraise_exception(): [ 904.758537] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 904.758922] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] self.force_reraise() [ 904.758922] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 904.758922] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] raise self.value [ 904.758922] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 904.758922] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] updated_port = self._update_port( [ 904.758922] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 904.758922] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] _ensure_no_port_binding_failure(port) [ 904.758922] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 904.758922] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] raise exception.PortBindingFailed(port_id=port['id']) [ 904.758922] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] nova.exception.PortBindingFailed: Binding failed for port aa3bd8c0-68d7-4a18-86d7-24b006731bea, please check neutron logs for more information. [ 904.758922] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] [ 904.758922] env[62499]: INFO nova.compute.manager [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Terminating instance [ 904.760076] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Acquiring lock "refresh_cache-d308417b-47d0-423a-a603-56d9251ef818" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 904.760245] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Acquired lock "refresh_cache-d308417b-47d0-423a-a603-56d9251ef818" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 904.760415] env[62499]: DEBUG nova.network.neutron [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 904.821631] env[62499]: DEBUG nova.network.neutron [-] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 905.017360] env[62499]: DEBUG oslo_concurrency.lockutils [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.333s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 905.017902] env[62499]: DEBUG nova.compute.manager [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 905.020526] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 21.705s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 905.282838] env[62499]: DEBUG nova.network.neutron [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 905.324178] env[62499]: INFO nova.compute.manager [-] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Took 1.02 seconds to deallocate network for instance. [ 905.327665] env[62499]: DEBUG nova.compute.claims [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 905.327863] env[62499]: DEBUG oslo_concurrency.lockutils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 905.361301] env[62499]: DEBUG nova.network.neutron [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 905.524761] env[62499]: DEBUG nova.compute.utils [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 905.528787] env[62499]: DEBUG nova.compute.manager [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Not allocating networking since 'none' was specified. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 905.680601] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d53eb8e-add3-4d8f-8102-059c5da2c208 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.688038] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-31fa80d4-0105-46bd-9793-344086dc6732 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.717311] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f30302ab-ef22-4fa5-89a3-5b41cd8e6965 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.725054] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ccb8b9b6-e05c-4457-98ad-af948f84c9dc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.738095] env[62499]: DEBUG nova.compute.provider_tree [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 905.864204] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Releasing lock "refresh_cache-d308417b-47d0-423a-a603-56d9251ef818" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 905.864592] env[62499]: DEBUG nova.compute.manager [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 905.864781] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 905.865092] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e6f56e43-5eb6-4f9c-9957-75069920ae8e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.874268] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76b87232-cb07-44bc-89fd-bf08d3fd736a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.895906] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d308417b-47d0-423a-a603-56d9251ef818 could not be found. [ 905.896118] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 905.896298] env[62499]: INFO nova.compute.manager [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Took 0.03 seconds to destroy the instance on the hypervisor. [ 905.896525] env[62499]: DEBUG oslo.service.loopingcall [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 905.896728] env[62499]: DEBUG nova.compute.manager [-] [instance: d308417b-47d0-423a-a603-56d9251ef818] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 905.896820] env[62499]: DEBUG nova.network.neutron [-] [instance: d308417b-47d0-423a-a603-56d9251ef818] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 905.911719] env[62499]: DEBUG nova.network.neutron [-] [instance: d308417b-47d0-423a-a603-56d9251ef818] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 906.031059] env[62499]: DEBUG nova.compute.manager [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 906.130619] env[62499]: DEBUG nova.compute.manager [req-ed8b478b-eea2-4ab9-bdad-d06999f08a40 req-d79b6917-48ed-423f-ab79-a0b887a38a6b service nova] [instance: d308417b-47d0-423a-a603-56d9251ef818] Received event network-changed-aa3bd8c0-68d7-4a18-86d7-24b006731bea {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 906.130779] env[62499]: DEBUG nova.compute.manager [req-ed8b478b-eea2-4ab9-bdad-d06999f08a40 req-d79b6917-48ed-423f-ab79-a0b887a38a6b service nova] [instance: d308417b-47d0-423a-a603-56d9251ef818] Refreshing instance network info cache due to event network-changed-aa3bd8c0-68d7-4a18-86d7-24b006731bea. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 906.131068] env[62499]: DEBUG oslo_concurrency.lockutils [req-ed8b478b-eea2-4ab9-bdad-d06999f08a40 req-d79b6917-48ed-423f-ab79-a0b887a38a6b service nova] Acquiring lock "refresh_cache-d308417b-47d0-423a-a603-56d9251ef818" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 906.131340] env[62499]: DEBUG oslo_concurrency.lockutils [req-ed8b478b-eea2-4ab9-bdad-d06999f08a40 req-d79b6917-48ed-423f-ab79-a0b887a38a6b service nova] Acquired lock "refresh_cache-d308417b-47d0-423a-a603-56d9251ef818" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 906.131515] env[62499]: DEBUG nova.network.neutron [req-ed8b478b-eea2-4ab9-bdad-d06999f08a40 req-d79b6917-48ed-423f-ab79-a0b887a38a6b service nova] [instance: d308417b-47d0-423a-a603-56d9251ef818] Refreshing network info cache for port aa3bd8c0-68d7-4a18-86d7-24b006731bea {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 906.241602] env[62499]: DEBUG nova.scheduler.client.report [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 906.413946] env[62499]: DEBUG nova.network.neutron [-] [instance: d308417b-47d0-423a-a603-56d9251ef818] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 906.650182] env[62499]: DEBUG nova.network.neutron [req-ed8b478b-eea2-4ab9-bdad-d06999f08a40 req-d79b6917-48ed-423f-ab79-a0b887a38a6b service nova] [instance: d308417b-47d0-423a-a603-56d9251ef818] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 906.721910] env[62499]: DEBUG nova.network.neutron [req-ed8b478b-eea2-4ab9-bdad-d06999f08a40 req-d79b6917-48ed-423f-ab79-a0b887a38a6b service nova] [instance: d308417b-47d0-423a-a603-56d9251ef818] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 906.746195] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.726s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 906.746784] env[62499]: ERROR nova.compute.manager [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 2a8fbacb-6166-43d0-b9f4-1c05df641aad, please check neutron logs for more information. [ 906.746784] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Traceback (most recent call last): [ 906.746784] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 906.746784] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] self.driver.spawn(context, instance, image_meta, [ 906.746784] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 906.746784] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 906.746784] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 906.746784] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] vm_ref = self.build_virtual_machine(instance, [ 906.746784] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 906.746784] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] vif_infos = vmwarevif.get_vif_info(self._session, [ 906.746784] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 906.747130] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] for vif in network_info: [ 906.747130] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 906.747130] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] return self._sync_wrapper(fn, *args, **kwargs) [ 906.747130] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 906.747130] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] self.wait() [ 906.747130] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 906.747130] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] self[:] = self._gt.wait() [ 906.747130] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 906.747130] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] return self._exit_event.wait() [ 906.747130] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 906.747130] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] result = hub.switch() [ 906.747130] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 906.747130] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] return self.greenlet.switch() [ 906.747604] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 906.747604] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] result = function(*args, **kwargs) [ 906.747604] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 906.747604] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] return func(*args, **kwargs) [ 906.747604] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 906.747604] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] raise e [ 906.747604] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 906.747604] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] nwinfo = self.network_api.allocate_for_instance( [ 906.747604] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 906.747604] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] created_port_ids = self._update_ports_for_instance( [ 906.747604] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 906.747604] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] with excutils.save_and_reraise_exception(): [ 906.747604] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 906.748080] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] self.force_reraise() [ 906.748080] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 906.748080] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] raise self.value [ 906.748080] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 906.748080] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] updated_port = self._update_port( [ 906.748080] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 906.748080] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] _ensure_no_port_binding_failure(port) [ 906.748080] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 906.748080] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] raise exception.PortBindingFailed(port_id=port['id']) [ 906.748080] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] nova.exception.PortBindingFailed: Binding failed for port 2a8fbacb-6166-43d0-b9f4-1c05df641aad, please check neutron logs for more information. [ 906.748080] env[62499]: ERROR nova.compute.manager [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] [ 906.748399] env[62499]: DEBUG nova.compute.utils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Binding failed for port 2a8fbacb-6166-43d0-b9f4-1c05df641aad, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 906.748607] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 21.409s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 906.751580] env[62499]: DEBUG nova.compute.manager [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Build of instance ba8fc3e0-dae8-4910-858b-dab1de72089e was re-scheduled: Binding failed for port 2a8fbacb-6166-43d0-b9f4-1c05df641aad, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 906.752017] env[62499]: DEBUG nova.compute.manager [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 906.752249] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquiring lock "refresh_cache-ba8fc3e0-dae8-4910-858b-dab1de72089e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 906.752396] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquired lock "refresh_cache-ba8fc3e0-dae8-4910-858b-dab1de72089e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 906.752553] env[62499]: DEBUG nova.network.neutron [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 906.916413] env[62499]: INFO nova.compute.manager [-] [instance: d308417b-47d0-423a-a603-56d9251ef818] Took 1.02 seconds to deallocate network for instance. [ 906.918803] env[62499]: DEBUG nova.compute.claims [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 906.918984] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 907.039579] env[62499]: DEBUG nova.compute.manager [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 907.061957] env[62499]: DEBUG nova.virt.hardware [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 907.062289] env[62499]: DEBUG nova.virt.hardware [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 907.062486] env[62499]: DEBUG nova.virt.hardware [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 907.062677] env[62499]: DEBUG nova.virt.hardware [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 907.062824] env[62499]: DEBUG nova.virt.hardware [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 907.062967] env[62499]: DEBUG nova.virt.hardware [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 907.063187] env[62499]: DEBUG nova.virt.hardware [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 907.063346] env[62499]: DEBUG nova.virt.hardware [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 907.063507] env[62499]: DEBUG nova.virt.hardware [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 907.063667] env[62499]: DEBUG nova.virt.hardware [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 907.063833] env[62499]: DEBUG nova.virt.hardware [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 907.064715] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6cc34e60-d5ab-4d4e-aaf3-1d0d292604d6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.073087] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d71e363-ef8e-4866-b547-58d4f0bc96a9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.085993] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Instance VIF info [] {{(pid=62499) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 907.091614] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Creating folder: Project (7ed0ce48db954ddc8ec4461142dd0a6c). Parent ref: group-v285191. {{(pid=62499) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 907.091860] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-7a0ea088-d9bc-421d-98be-be46d09fb0a7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.101014] env[62499]: INFO nova.virt.vmwareapi.vm_util [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Created folder: Project (7ed0ce48db954ddc8ec4461142dd0a6c) in parent group-v285191. [ 907.101192] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Creating folder: Instances. Parent ref: group-v285220. {{(pid=62499) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 907.101386] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-2308bfac-f93a-445f-85f2-20673e7a0c15 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.109949] env[62499]: INFO nova.virt.vmwareapi.vm_util [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Created folder: Instances in parent group-v285220. [ 907.110177] env[62499]: DEBUG oslo.service.loopingcall [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 907.110355] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Creating VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 907.110533] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-cd846999-013c-4565-8dee-3c6a827510d0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.127417] env[62499]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 907.127417] env[62499]: value = "task-1335771" [ 907.127417] env[62499]: _type = "Task" [ 907.127417] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 907.134168] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335771, 'name': CreateVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 907.224461] env[62499]: DEBUG oslo_concurrency.lockutils [req-ed8b478b-eea2-4ab9-bdad-d06999f08a40 req-d79b6917-48ed-423f-ab79-a0b887a38a6b service nova] Releasing lock "refresh_cache-d308417b-47d0-423a-a603-56d9251ef818" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 907.224788] env[62499]: DEBUG nova.compute.manager [req-ed8b478b-eea2-4ab9-bdad-d06999f08a40 req-d79b6917-48ed-423f-ab79-a0b887a38a6b service nova] [instance: d308417b-47d0-423a-a603-56d9251ef818] Received event network-vif-deleted-aa3bd8c0-68d7-4a18-86d7-24b006731bea {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 907.273911] env[62499]: DEBUG nova.network.neutron [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 907.350654] env[62499]: DEBUG nova.network.neutron [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 907.425191] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-383a0739-e210-46c8-94d2-1adf631bf2ef {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.432713] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c2605755-3636-4192-984c-0462a40b8c64 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.462042] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8bf1ad1-4a5d-4280-8e07-640c15041d78 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.468220] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-65523ed6-9325-467e-b211-0319aa4304f4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.480639] env[62499]: DEBUG nova.compute.provider_tree [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 907.637426] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335771, 'name': CreateVM_Task, 'duration_secs': 0.248117} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 907.637598] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Created VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 907.638052] env[62499]: DEBUG oslo_concurrency.lockutils [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 907.638219] env[62499]: DEBUG oslo_concurrency.lockutils [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquired lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 907.638556] env[62499]: DEBUG oslo_concurrency.lockutils [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 907.638803] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-9506027a-500d-4c9b-ad3b-5fc212768903 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.643627] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 907.643627] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52188d7a-82ea-cf0d-33ab-db1ebdddeac2" [ 907.643627] env[62499]: _type = "Task" [ 907.643627] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 907.651091] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52188d7a-82ea-cf0d-33ab-db1ebdddeac2, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 907.852937] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Releasing lock "refresh_cache-ba8fc3e0-dae8-4910-858b-dab1de72089e" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 907.853232] env[62499]: DEBUG nova.compute.manager [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 907.853432] env[62499]: DEBUG nova.compute.manager [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 907.853604] env[62499]: DEBUG nova.network.neutron [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 907.867157] env[62499]: DEBUG nova.network.neutron [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 907.983537] env[62499]: DEBUG nova.scheduler.client.report [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 908.154406] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52188d7a-82ea-cf0d-33ab-db1ebdddeac2, 'name': SearchDatastore_Task, 'duration_secs': 0.009099} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 908.154754] env[62499]: DEBUG oslo_concurrency.lockutils [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Releasing lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 908.155036] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Processing image 0422123c-16ce-4c9d-b1c6-0281fe0e54ad {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 908.155341] env[62499]: DEBUG oslo_concurrency.lockutils [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 908.155524] env[62499]: DEBUG oslo_concurrency.lockutils [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquired lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 908.155732] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 908.156012] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-f755232d-683c-40de-95ef-0c8c5bb211b3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.163249] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 908.163422] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62499) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 908.164079] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-9581f9e9-b1bf-4689-9f7f-4ac373ead69f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.168462] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 908.168462] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52c9e5fe-c9ff-d9db-6718-342670190bfc" [ 908.168462] env[62499]: _type = "Task" [ 908.168462] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 908.175664] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52c9e5fe-c9ff-d9db-6718-342670190bfc, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 908.370030] env[62499]: DEBUG nova.network.neutron [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 908.489076] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.740s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 908.489553] env[62499]: ERROR nova.compute.manager [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 4681981e-8791-4c18-b0e8-49b24f59baa0, please check neutron logs for more information. [ 908.489553] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Traceback (most recent call last): [ 908.489553] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 908.489553] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] self.driver.spawn(context, instance, image_meta, [ 908.489553] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 908.489553] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] self._vmops.spawn(context, instance, image_meta, injected_files, [ 908.489553] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 908.489553] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] vm_ref = self.build_virtual_machine(instance, [ 908.489553] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 908.489553] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] vif_infos = vmwarevif.get_vif_info(self._session, [ 908.489553] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 908.489947] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] for vif in network_info: [ 908.489947] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 908.489947] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] return self._sync_wrapper(fn, *args, **kwargs) [ 908.489947] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 908.489947] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] self.wait() [ 908.489947] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 908.489947] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] self[:] = self._gt.wait() [ 908.489947] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 908.489947] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] return self._exit_event.wait() [ 908.489947] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 908.489947] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] current.throw(*self._exc) [ 908.489947] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 908.489947] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] result = function(*args, **kwargs) [ 908.490343] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 908.490343] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] return func(*args, **kwargs) [ 908.490343] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 908.490343] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] raise e [ 908.490343] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 908.490343] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] nwinfo = self.network_api.allocate_for_instance( [ 908.490343] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 908.490343] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] created_port_ids = self._update_ports_for_instance( [ 908.490343] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 908.490343] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] with excutils.save_and_reraise_exception(): [ 908.490343] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 908.490343] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] self.force_reraise() [ 908.490343] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 908.490761] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] raise self.value [ 908.490761] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 908.490761] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] updated_port = self._update_port( [ 908.490761] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 908.490761] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] _ensure_no_port_binding_failure(port) [ 908.490761] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 908.490761] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] raise exception.PortBindingFailed(port_id=port['id']) [ 908.490761] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] nova.exception.PortBindingFailed: Binding failed for port 4681981e-8791-4c18-b0e8-49b24f59baa0, please check neutron logs for more information. [ 908.490761] env[62499]: ERROR nova.compute.manager [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] [ 908.490761] env[62499]: DEBUG nova.compute.utils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Binding failed for port 4681981e-8791-4c18-b0e8-49b24f59baa0, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 908.492023] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 20.644s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 908.494686] env[62499]: DEBUG nova.compute.manager [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Build of instance 0eaadb76-e2c7-4e06-b865-065ab0f014ba was re-scheduled: Binding failed for port 4681981e-8791-4c18-b0e8-49b24f59baa0, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 908.495110] env[62499]: DEBUG nova.compute.manager [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 908.495333] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquiring lock "refresh_cache-0eaadb76-e2c7-4e06-b865-065ab0f014ba" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 908.495477] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Acquired lock "refresh_cache-0eaadb76-e2c7-4e06-b865-065ab0f014ba" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 908.495633] env[62499]: DEBUG nova.network.neutron [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 908.678910] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52c9e5fe-c9ff-d9db-6718-342670190bfc, 'name': SearchDatastore_Task, 'duration_secs': 0.008102} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 908.679665] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-2245c6f0-1142-4293-889b-ec4363a3321d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.684564] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 908.684564] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52429be0-968f-d8e7-0a6e-28ff0a11d3fd" [ 908.684564] env[62499]: _type = "Task" [ 908.684564] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 908.692293] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52429be0-968f-d8e7-0a6e-28ff0a11d3fd, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 908.872742] env[62499]: INFO nova.compute.manager [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: ba8fc3e0-dae8-4910-858b-dab1de72089e] Took 1.02 seconds to deallocate network for instance. [ 909.015428] env[62499]: DEBUG nova.network.neutron [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 909.085607] env[62499]: DEBUG nova.network.neutron [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 909.160027] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90f93fec-b25d-421e-9e1b-d92524d072f7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 909.167525] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4a4a862-4cea-469d-bccf-88a4dfafc3cf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 909.200870] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ca37081-034f-4f2d-8f91-da01a8706964 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 909.210904] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7b911ade-086f-4bbe-97ea-baf198ef1f46 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 909.214347] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52429be0-968f-d8e7-0a6e-28ff0a11d3fd, 'name': SearchDatastore_Task, 'duration_secs': 0.008799} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 909.214595] env[62499]: DEBUG oslo_concurrency.lockutils [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Releasing lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 909.214842] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore2] b534fcdb-b0db-4e07-85fc-696dc58168aa/b534fcdb-b0db-4e07-85fc-696dc58168aa.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 909.215412] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-546df5d1-7706-47a0-86b0-e257de483589 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 909.224529] env[62499]: DEBUG nova.compute.provider_tree [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 909.229743] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 909.229743] env[62499]: value = "task-1335772" [ 909.229743] env[62499]: _type = "Task" [ 909.229743] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 909.237272] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335772, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 909.591071] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Releasing lock "refresh_cache-0eaadb76-e2c7-4e06-b865-065ab0f014ba" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 909.591304] env[62499]: DEBUG nova.compute.manager [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 909.591501] env[62499]: DEBUG nova.compute.manager [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 909.591676] env[62499]: DEBUG nova.network.neutron [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 909.608181] env[62499]: DEBUG nova.network.neutron [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 909.727266] env[62499]: DEBUG nova.scheduler.client.report [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 909.739032] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335772, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.487318} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 909.739264] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore2] b534fcdb-b0db-4e07-85fc-696dc58168aa/b534fcdb-b0db-4e07-85fc-696dc58168aa.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 909.739475] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Extending root virtual disk to 1048576 {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 909.739708] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-fe0159ec-21e8-440a-b4c7-257d87f36767 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 909.746404] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 909.746404] env[62499]: value = "task-1335773" [ 909.746404] env[62499]: _type = "Task" [ 909.746404] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 909.754157] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335773, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 909.900662] env[62499]: INFO nova.scheduler.client.report [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Deleted allocations for instance ba8fc3e0-dae8-4910-858b-dab1de72089e [ 910.111288] env[62499]: DEBUG nova.network.neutron [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 910.235811] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.744s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 910.236450] env[62499]: ERROR nova.compute.manager [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 9a754f05-ea5f-4418-aa84-0078099e5dce, please check neutron logs for more information. [ 910.236450] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Traceback (most recent call last): [ 910.236450] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 910.236450] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] self.driver.spawn(context, instance, image_meta, [ 910.236450] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 910.236450] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 910.236450] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 910.236450] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] vm_ref = self.build_virtual_machine(instance, [ 910.236450] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 910.236450] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] vif_infos = vmwarevif.get_vif_info(self._session, [ 910.236450] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 910.236848] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] for vif in network_info: [ 910.236848] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 910.236848] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] return self._sync_wrapper(fn, *args, **kwargs) [ 910.236848] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 910.236848] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] self.wait() [ 910.236848] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 910.236848] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] self[:] = self._gt.wait() [ 910.236848] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 910.236848] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] return self._exit_event.wait() [ 910.236848] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 910.236848] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] current.throw(*self._exc) [ 910.236848] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 910.236848] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] result = function(*args, **kwargs) [ 910.237237] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 910.237237] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] return func(*args, **kwargs) [ 910.237237] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 910.237237] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] raise e [ 910.237237] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 910.237237] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] nwinfo = self.network_api.allocate_for_instance( [ 910.237237] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 910.237237] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] created_port_ids = self._update_ports_for_instance( [ 910.237237] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 910.237237] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] with excutils.save_and_reraise_exception(): [ 910.237237] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 910.237237] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] self.force_reraise() [ 910.237237] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 910.237609] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] raise self.value [ 910.237609] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 910.237609] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] updated_port = self._update_port( [ 910.237609] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 910.237609] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] _ensure_no_port_binding_failure(port) [ 910.237609] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 910.237609] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] raise exception.PortBindingFailed(port_id=port['id']) [ 910.237609] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] nova.exception.PortBindingFailed: Binding failed for port 9a754f05-ea5f-4418-aa84-0078099e5dce, please check neutron logs for more information. [ 910.237609] env[62499]: ERROR nova.compute.manager [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] [ 910.237609] env[62499]: DEBUG nova.compute.utils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Binding failed for port 9a754f05-ea5f-4418-aa84-0078099e5dce, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 910.238888] env[62499]: DEBUG nova.compute.manager [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Build of instance 764ef9ab-22ae-45cd-8b5d-76a63346d93c was re-scheduled: Binding failed for port 9a754f05-ea5f-4418-aa84-0078099e5dce, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 910.239298] env[62499]: DEBUG nova.compute.manager [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 910.239524] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "refresh_cache-764ef9ab-22ae-45cd-8b5d-76a63346d93c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 910.239671] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquired lock "refresh_cache-764ef9ab-22ae-45cd-8b5d-76a63346d93c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 910.239830] env[62499]: DEBUG nova.network.neutron [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 910.240779] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 20.964s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 910.256694] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335773, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.060364} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 910.256923] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Extended root virtual disk {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 910.257651] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86d391ab-7c34-4328-8ebd-38a90c7a61e5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.276541] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Reconfiguring VM instance instance-0000004e to attach disk [datastore2] b534fcdb-b0db-4e07-85fc-696dc58168aa/b534fcdb-b0db-4e07-85fc-696dc58168aa.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 910.277304] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-f5b5df97-6686-4b52-bd9f-e1286aec0b4d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.296372] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 910.296372] env[62499]: value = "task-1335774" [ 910.296372] env[62499]: _type = "Task" [ 910.296372] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 910.303749] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335774, 'name': ReconfigVM_Task} progress is 5%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 910.408761] env[62499]: DEBUG oslo_concurrency.lockutils [None req-d99fa83e-f044-4de4-a72d-022ee6946c23 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "ba8fc3e0-dae8-4910-858b-dab1de72089e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 162.163s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 910.613555] env[62499]: INFO nova.compute.manager [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] [instance: 0eaadb76-e2c7-4e06-b865-065ab0f014ba] Took 1.02 seconds to deallocate network for instance. [ 910.763178] env[62499]: DEBUG nova.network.neutron [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 910.811704] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335774, 'name': ReconfigVM_Task, 'duration_secs': 0.262827} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 910.813966] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Reconfigured VM instance instance-0000004e to attach disk [datastore2] b534fcdb-b0db-4e07-85fc-696dc58168aa/b534fcdb-b0db-4e07-85fc-696dc58168aa.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 910.816244] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-2af978b7-b1dc-4ff1-9b83-a05d5514425f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.826023] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 910.826023] env[62499]: value = "task-1335775" [ 910.826023] env[62499]: _type = "Task" [ 910.826023] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 910.833942] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335775, 'name': Rename_Task} progress is 5%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 910.877333] env[62499]: DEBUG nova.network.neutron [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 910.913980] env[62499]: DEBUG nova.compute.manager [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 910.942383] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec52d8df-e54a-4c9a-a0f4-328685bfa408 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.950223] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0543d49f-748f-4f74-aaa1-915367ab17a3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.980871] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74768c8b-7c4d-43d8-bf27-b4003dd1b57c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.988729] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-427ac6a3-9132-4577-ac24-e06f375c37de {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 911.001697] env[62499]: DEBUG nova.compute.provider_tree [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 911.334251] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335775, 'name': Rename_Task, 'duration_secs': 0.129425} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 911.334510] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Powering on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 911.334742] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-036dff85-2964-485b-a9a0-559f5788412e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 911.340435] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 911.340435] env[62499]: value = "task-1335776" [ 911.340435] env[62499]: _type = "Task" [ 911.340435] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 911.347573] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335776, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 911.379463] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Releasing lock "refresh_cache-764ef9ab-22ae-45cd-8b5d-76a63346d93c" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 911.379704] env[62499]: DEBUG nova.compute.manager [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 911.379886] env[62499]: DEBUG nova.compute.manager [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 911.380150] env[62499]: DEBUG nova.network.neutron [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 911.396610] env[62499]: DEBUG nova.network.neutron [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 911.434253] env[62499]: DEBUG oslo_concurrency.lockutils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 911.505378] env[62499]: DEBUG nova.scheduler.client.report [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 911.638453] env[62499]: INFO nova.scheduler.client.report [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Deleted allocations for instance 0eaadb76-e2c7-4e06-b865-065ab0f014ba [ 911.850676] env[62499]: DEBUG oslo_vmware.api [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335776, 'name': PowerOnVM_Task, 'duration_secs': 0.416224} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 911.851117] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Powered on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 911.851263] env[62499]: INFO nova.compute.manager [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Took 4.81 seconds to spawn the instance on the hypervisor. [ 911.851451] env[62499]: DEBUG nova.compute.manager [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 911.852220] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0f709f9-5418-4cb2-ad0b-0d03a2844ebb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 911.899574] env[62499]: DEBUG nova.network.neutron [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 912.010657] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.770s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 912.011311] env[62499]: ERROR nova.compute.manager [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 0349f93a-0a30-49e4-a1c6-0c72ee6a71c9, please check neutron logs for more information. [ 912.011311] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Traceback (most recent call last): [ 912.011311] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 912.011311] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] self.driver.spawn(context, instance, image_meta, [ 912.011311] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 912.011311] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 912.011311] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 912.011311] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] vm_ref = self.build_virtual_machine(instance, [ 912.011311] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 912.011311] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] vif_infos = vmwarevif.get_vif_info(self._session, [ 912.011311] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 912.011627] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] for vif in network_info: [ 912.011627] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 912.011627] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] return self._sync_wrapper(fn, *args, **kwargs) [ 912.011627] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 912.011627] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] self.wait() [ 912.011627] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 912.011627] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] self[:] = self._gt.wait() [ 912.011627] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 912.011627] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] return self._exit_event.wait() [ 912.011627] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 912.011627] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] current.throw(*self._exc) [ 912.011627] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 912.011627] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] result = function(*args, **kwargs) [ 912.011948] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 912.011948] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] return func(*args, **kwargs) [ 912.011948] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 912.011948] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] raise e [ 912.011948] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 912.011948] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] nwinfo = self.network_api.allocate_for_instance( [ 912.011948] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 912.011948] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] created_port_ids = self._update_ports_for_instance( [ 912.011948] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 912.011948] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] with excutils.save_and_reraise_exception(): [ 912.011948] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 912.011948] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] self.force_reraise() [ 912.011948] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 912.012284] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] raise self.value [ 912.012284] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 912.012284] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] updated_port = self._update_port( [ 912.012284] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 912.012284] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] _ensure_no_port_binding_failure(port) [ 912.012284] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 912.012284] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] raise exception.PortBindingFailed(port_id=port['id']) [ 912.012284] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] nova.exception.PortBindingFailed: Binding failed for port 0349f93a-0a30-49e4-a1c6-0c72ee6a71c9, please check neutron logs for more information. [ 912.012284] env[62499]: ERROR nova.compute.manager [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] [ 912.012284] env[62499]: DEBUG nova.compute.utils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Binding failed for port 0349f93a-0a30-49e4-a1c6-0c72ee6a71c9, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 912.013825] env[62499]: DEBUG oslo_concurrency.lockutils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 20.389s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 912.016214] env[62499]: DEBUG nova.compute.manager [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Build of instance a92509e3-d5b1-41d3-b9af-fa0af8a956c9 was re-scheduled: Binding failed for port 0349f93a-0a30-49e4-a1c6-0c72ee6a71c9, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 912.016683] env[62499]: DEBUG nova.compute.manager [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 912.016913] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "refresh_cache-a92509e3-d5b1-41d3-b9af-fa0af8a956c9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 912.017104] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquired lock "refresh_cache-a92509e3-d5b1-41d3-b9af-fa0af8a956c9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 912.017225] env[62499]: DEBUG nova.network.neutron [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 912.147048] env[62499]: DEBUG oslo_concurrency.lockutils [None req-c09994b6-9b80-4010-907a-2b3984b40ed2 tempest-ListServerFiltersTestJSON-1553140998 tempest-ListServerFiltersTestJSON-1553140998-project-member] Lock "0eaadb76-e2c7-4e06-b865-065ab0f014ba" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 163.263s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 912.368767] env[62499]: INFO nova.compute.manager [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Took 33.68 seconds to build instance. [ 912.402182] env[62499]: INFO nova.compute.manager [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 764ef9ab-22ae-45cd-8b5d-76a63346d93c] Took 1.02 seconds to deallocate network for instance. [ 912.554968] env[62499]: DEBUG nova.network.neutron [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 912.649016] env[62499]: DEBUG nova.compute.manager [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 912.692187] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67f7bea6-ed12-4547-99b2-a240af65a93f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 912.700535] env[62499]: DEBUG nova.network.neutron [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 912.702373] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d63e90aa-cfef-4e67-ac1c-bf75a4bd3123 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 912.733860] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-04389b7c-e562-4dba-bf0f-9053744e8cc8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 912.741792] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e9ae197a-5b14-4127-95c2-4121074d6388 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 912.755704] env[62499]: DEBUG nova.compute.provider_tree [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 912.872176] env[62499]: DEBUG oslo_concurrency.lockutils [None req-841b55b1-1ace-4c65-9409-c7753aa92bbb tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Lock "b534fcdb-b0db-4e07-85fc-696dc58168aa" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 53.809s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 913.030886] env[62499]: INFO nova.compute.manager [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Rebuilding instance [ 913.073680] env[62499]: DEBUG nova.compute.manager [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 913.074541] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-06aa6996-f91d-41c0-a7d9-d26677c4ac30 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 913.170864] env[62499]: DEBUG oslo_concurrency.lockutils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 913.207101] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Releasing lock "refresh_cache-a92509e3-d5b1-41d3-b9af-fa0af8a956c9" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 913.207356] env[62499]: DEBUG nova.compute.manager [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 913.207538] env[62499]: DEBUG nova.compute.manager [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 913.207703] env[62499]: DEBUG nova.network.neutron [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 913.221149] env[62499]: DEBUG nova.network.neutron [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 913.258946] env[62499]: DEBUG nova.scheduler.client.report [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 913.379022] env[62499]: DEBUG nova.compute.manager [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 913.430024] env[62499]: INFO nova.scheduler.client.report [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Deleted allocations for instance 764ef9ab-22ae-45cd-8b5d-76a63346d93c [ 913.584897] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Powering off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 913.585219] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-dd51e9a3-a03c-4236-9ad5-69131e3ca87a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 913.592477] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 913.592477] env[62499]: value = "task-1335777" [ 913.592477] env[62499]: _type = "Task" [ 913.592477] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 913.600126] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335777, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 913.724031] env[62499]: DEBUG nova.network.neutron [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 913.765134] env[62499]: DEBUG oslo_concurrency.lockutils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.751s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 913.765824] env[62499]: ERROR nova.compute.manager [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 982d43f5-d2d0-473c-a37b-1cb4ec42ab11, please check neutron logs for more information. [ 913.765824] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] Traceback (most recent call last): [ 913.765824] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 913.765824] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] self.driver.spawn(context, instance, image_meta, [ 913.765824] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 913.765824] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 913.765824] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 913.765824] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] vm_ref = self.build_virtual_machine(instance, [ 913.765824] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 913.765824] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] vif_infos = vmwarevif.get_vif_info(self._session, [ 913.765824] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 913.766193] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] for vif in network_info: [ 913.766193] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 913.766193] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] return self._sync_wrapper(fn, *args, **kwargs) [ 913.766193] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 913.766193] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] self.wait() [ 913.766193] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 913.766193] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] self[:] = self._gt.wait() [ 913.766193] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 913.766193] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] return self._exit_event.wait() [ 913.766193] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 913.766193] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] current.throw(*self._exc) [ 913.766193] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 913.766193] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] result = function(*args, **kwargs) [ 913.766586] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 913.766586] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] return func(*args, **kwargs) [ 913.766586] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 913.766586] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] raise e [ 913.766586] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 913.766586] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] nwinfo = self.network_api.allocate_for_instance( [ 913.766586] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 913.766586] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] created_port_ids = self._update_ports_for_instance( [ 913.766586] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 913.766586] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] with excutils.save_and_reraise_exception(): [ 913.766586] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 913.766586] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] self.force_reraise() [ 913.766586] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 913.766965] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] raise self.value [ 913.766965] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 913.766965] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] updated_port = self._update_port( [ 913.766965] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 913.766965] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] _ensure_no_port_binding_failure(port) [ 913.766965] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 913.766965] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] raise exception.PortBindingFailed(port_id=port['id']) [ 913.766965] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] nova.exception.PortBindingFailed: Binding failed for port 982d43f5-d2d0-473c-a37b-1cb4ec42ab11, please check neutron logs for more information. [ 913.766965] env[62499]: ERROR nova.compute.manager [instance: 770160d6-4681-40e0-a812-5c491b804da4] [ 913.766965] env[62499]: DEBUG nova.compute.utils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Binding failed for port 982d43f5-d2d0-473c-a37b-1cb4ec42ab11, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 913.768276] env[62499]: DEBUG nova.compute.manager [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Build of instance 770160d6-4681-40e0-a812-5c491b804da4 was re-scheduled: Binding failed for port 982d43f5-d2d0-473c-a37b-1cb4ec42ab11, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 913.768733] env[62499]: DEBUG nova.compute.manager [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 913.768999] env[62499]: DEBUG oslo_concurrency.lockutils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquiring lock "refresh_cache-770160d6-4681-40e0-a812-5c491b804da4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 913.769201] env[62499]: DEBUG oslo_concurrency.lockutils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquired lock "refresh_cache-770160d6-4681-40e0-a812-5c491b804da4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 913.769405] env[62499]: DEBUG nova.network.neutron [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 913.770975] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 19.864s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 913.900525] env[62499]: DEBUG oslo_concurrency.lockutils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 913.938033] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b73f9761-6eed-4f72-b930-96fb390553b6 tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "764ef9ab-22ae-45cd-8b5d-76a63346d93c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 153.373s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 914.102187] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335777, 'name': PowerOffVM_Task, 'duration_secs': 0.111244} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 914.102466] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Powered off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 914.102679] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 914.103452] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ce25dab-d735-40b7-a4b8-31d3ef0b431d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 914.110015] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Unregistering the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 914.110484] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-fa0cb7d1-651b-4600-aa65-1b29b220a409 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 914.133604] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Unregistered the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 914.133822] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Deleting contents of the VM from datastore datastore2 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 914.134111] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Deleting the datastore file [datastore2] b534fcdb-b0db-4e07-85fc-696dc58168aa {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 914.134267] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-6caa001f-d337-42b5-b3c8-22ac91ab495f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 914.140327] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 914.140327] env[62499]: value = "task-1335779" [ 914.140327] env[62499]: _type = "Task" [ 914.140327] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 914.148114] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335779, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 914.227734] env[62499]: INFO nova.compute.manager [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: a92509e3-d5b1-41d3-b9af-fa0af8a956c9] Took 1.02 seconds to deallocate network for instance. [ 914.293060] env[62499]: DEBUG nova.network.neutron [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 914.383327] env[62499]: DEBUG nova.network.neutron [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 914.437885] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1148ffa9-6893-49f4-8e87-120a01b657e9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 914.441445] env[62499]: DEBUG nova.compute.manager [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 914.450991] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02ecbd14-cfc3-406f-9196-7285925ff19f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 914.487683] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08cad117-0ccf-4b02-a21f-9e223896a295 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 914.495498] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-957d00f2-ef07-4b54-96b1-6eccd31ead36 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 914.509188] env[62499]: DEBUG nova.compute.provider_tree [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 914.649780] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335779, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.149277} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 914.650058] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Deleted the datastore file {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 914.650250] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Deleted contents of the VM from datastore datastore2 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 914.650425] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 914.818625] env[62499]: DEBUG oslo_concurrency.lockutils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "9fd7db7e-466e-4213-9e1d-210eae714884" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 914.818625] env[62499]: DEBUG oslo_concurrency.lockutils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "9fd7db7e-466e-4213-9e1d-210eae714884" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 914.890051] env[62499]: DEBUG oslo_concurrency.lockutils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Releasing lock "refresh_cache-770160d6-4681-40e0-a812-5c491b804da4" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 914.890051] env[62499]: DEBUG nova.compute.manager [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 914.890051] env[62499]: DEBUG nova.compute.manager [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 914.890312] env[62499]: DEBUG nova.network.neutron [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 914.907379] env[62499]: DEBUG nova.network.neutron [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 914.961489] env[62499]: DEBUG oslo_concurrency.lockutils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 915.012989] env[62499]: DEBUG nova.scheduler.client.report [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 915.251984] env[62499]: INFO nova.scheduler.client.report [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Deleted allocations for instance a92509e3-d5b1-41d3-b9af-fa0af8a956c9 [ 915.410373] env[62499]: DEBUG nova.network.neutron [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 915.517517] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.746s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 915.518172] env[62499]: ERROR nova.compute.manager [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 765be7c7-1b25-4959-b478-d5f415357706, please check neutron logs for more information. [ 915.518172] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Traceback (most recent call last): [ 915.518172] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 915.518172] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] self.driver.spawn(context, instance, image_meta, [ 915.518172] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 915.518172] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 915.518172] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 915.518172] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] vm_ref = self.build_virtual_machine(instance, [ 915.518172] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 915.518172] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] vif_infos = vmwarevif.get_vif_info(self._session, [ 915.518172] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 915.518561] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] for vif in network_info: [ 915.518561] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 915.518561] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] return self._sync_wrapper(fn, *args, **kwargs) [ 915.518561] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 915.518561] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] self.wait() [ 915.518561] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 915.518561] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] self[:] = self._gt.wait() [ 915.518561] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 915.518561] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] return self._exit_event.wait() [ 915.518561] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 915.518561] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] current.throw(*self._exc) [ 915.518561] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 915.518561] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] result = function(*args, **kwargs) [ 915.518890] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 915.518890] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] return func(*args, **kwargs) [ 915.518890] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 915.518890] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] raise e [ 915.518890] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 915.518890] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] nwinfo = self.network_api.allocate_for_instance( [ 915.518890] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 915.518890] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] created_port_ids = self._update_ports_for_instance( [ 915.518890] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 915.518890] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] with excutils.save_and_reraise_exception(): [ 915.518890] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 915.518890] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] self.force_reraise() [ 915.518890] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 915.519214] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] raise self.value [ 915.519214] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 915.519214] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] updated_port = self._update_port( [ 915.519214] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 915.519214] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] _ensure_no_port_binding_failure(port) [ 915.519214] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 915.519214] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] raise exception.PortBindingFailed(port_id=port['id']) [ 915.519214] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] nova.exception.PortBindingFailed: Binding failed for port 765be7c7-1b25-4959-b478-d5f415357706, please check neutron logs for more information. [ 915.519214] env[62499]: ERROR nova.compute.manager [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] [ 915.519214] env[62499]: DEBUG nova.compute.utils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Binding failed for port 765be7c7-1b25-4959-b478-d5f415357706, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 915.520490] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 19.412s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 915.522012] env[62499]: INFO nova.compute.claims [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 915.524608] env[62499]: DEBUG nova.compute.manager [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Build of instance cbe51201-c036-4f90-9372-8cc2d033f7a2 was re-scheduled: Binding failed for port 765be7c7-1b25-4959-b478-d5f415357706, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 915.525398] env[62499]: DEBUG nova.compute.manager [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 915.525398] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Acquiring lock "refresh_cache-cbe51201-c036-4f90-9372-8cc2d033f7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 915.525564] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Acquired lock "refresh_cache-cbe51201-c036-4f90-9372-8cc2d033f7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 915.525649] env[62499]: DEBUG nova.network.neutron [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 915.686244] env[62499]: DEBUG nova.virt.hardware [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 915.686522] env[62499]: DEBUG nova.virt.hardware [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 915.686687] env[62499]: DEBUG nova.virt.hardware [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 915.686860] env[62499]: DEBUG nova.virt.hardware [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 915.687026] env[62499]: DEBUG nova.virt.hardware [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 915.687162] env[62499]: DEBUG nova.virt.hardware [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 915.687362] env[62499]: DEBUG nova.virt.hardware [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 915.687517] env[62499]: DEBUG nova.virt.hardware [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 915.687675] env[62499]: DEBUG nova.virt.hardware [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 915.687856] env[62499]: DEBUG nova.virt.hardware [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 915.688062] env[62499]: DEBUG nova.virt.hardware [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 915.689235] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a5bd4b6c-df39-405b-9c46-ded4313e65ae {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 915.697730] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4b644cc-4921-4320-b0a8-5145a4b975d5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 915.711076] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Instance VIF info [] {{(pid=62499) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 915.716648] env[62499]: DEBUG oslo.service.loopingcall [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 915.716956] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Creating VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1337}} [ 915.717104] env[62499]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-9365b8b9-df40-4913-8353-128cedfba0d5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 915.734855] env[62499]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 915.734855] env[62499]: value = "task-1335780" [ 915.734855] env[62499]: _type = "Task" [ 915.734855] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 915.759812] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ebf251d1-e02e-4f79-b4f9-ee2ca970b777 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "a92509e3-d5b1-41d3-b9af-fa0af8a956c9" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 136.128s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 915.913105] env[62499]: INFO nova.compute.manager [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 770160d6-4681-40e0-a812-5c491b804da4] Took 1.02 seconds to deallocate network for instance. [ 916.047426] env[62499]: DEBUG nova.network.neutron [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 916.129917] env[62499]: DEBUG nova.network.neutron [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 916.244294] env[62499]: DEBUG oslo_vmware.api [-] Task: {'id': task-1335780, 'name': CreateVM_Task, 'duration_secs': 0.324661} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 916.244456] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Created VM on the ESX host {{(pid=62499) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1359}} [ 916.244848] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 916.245015] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquired lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 916.245333] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 916.245570] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-2b02dc46-64fb-45f8-b67a-9afd29a45c66 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.249701] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 916.249701] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]5283246c-ad74-0122-091b-45f51296c886" [ 916.249701] env[62499]: _type = "Task" [ 916.249701] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 916.256665] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]5283246c-ad74-0122-091b-45f51296c886, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 916.262115] env[62499]: DEBUG nova.compute.manager [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 916.632813] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Releasing lock "refresh_cache-cbe51201-c036-4f90-9372-8cc2d033f7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 916.633104] env[62499]: DEBUG nova.compute.manager [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 916.633326] env[62499]: DEBUG nova.compute.manager [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 916.633535] env[62499]: DEBUG nova.network.neutron [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 916.652085] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "ea4b6c62-2880-4a59-8017-b5d1518c1060" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 916.652334] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "ea4b6c62-2880-4a59-8017-b5d1518c1060" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 916.653031] env[62499]: DEBUG nova.network.neutron [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 916.685039] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc38f058-fdff-4f4f-b21d-9f9372469d63 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.692311] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e10d6dd-a1df-44e8-969f-e4d037a3b3c9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.720968] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4cedd763-8d1e-4824-b47f-740547ebc54e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.727582] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1b7f9284-2523-47c9-8acd-a6b15e92324c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.742128] env[62499]: DEBUG nova.compute.provider_tree [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 916.758692] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]5283246c-ad74-0122-091b-45f51296c886, 'name': SearchDatastore_Task, 'duration_secs': 0.008636} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 916.758962] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Releasing lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 916.759202] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Processing image 0422123c-16ce-4c9d-b1c6-0281fe0e54ad {{(pid=62499) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 916.759422] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 916.759563] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquired lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 916.759734] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 916.759976] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-d9f7f3a1-2bda-45a8-a035-b6e803eef95c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.768778] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62499) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 916.768943] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62499) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1845}} [ 916.771289] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-4c0ea576-45d0-47df-b16b-8a270202932a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.775970] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 916.775970] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52497af1-40f1-a3e1-fa4d-45e2e6116300" [ 916.775970] env[62499]: _type = "Task" [ 916.775970] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 916.783081] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52497af1-40f1-a3e1-fa4d-45e2e6116300, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 916.783862] env[62499]: DEBUG oslo_concurrency.lockutils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 916.939171] env[62499]: INFO nova.scheduler.client.report [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Deleted allocations for instance 770160d6-4681-40e0-a812-5c491b804da4 [ 917.155793] env[62499]: DEBUG nova.network.neutron [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 917.245378] env[62499]: DEBUG nova.scheduler.client.report [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 917.286069] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52497af1-40f1-a3e1-fa4d-45e2e6116300, 'name': SearchDatastore_Task, 'duration_secs': 0.00762} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 917.286826] env[62499]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-da5928eb-504e-408f-b27b-4ea13cbc4b5c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 917.291821] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 917.291821] env[62499]: value = "session[5219b453-ddc7-8e28-3ba2-af7e13031566]52e119a1-3ada-b713-e2cb-d7f10e7ee82f" [ 917.291821] env[62499]: _type = "Task" [ 917.291821] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 917.299375] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52e119a1-3ada-b713-e2cb-d7f10e7ee82f, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 917.447017] env[62499]: DEBUG oslo_concurrency.lockutils [None req-7aeb3b99-c914-4c2d-8ca6-8bdb31a653d6 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "770160d6-4681-40e0-a812-5c491b804da4" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 113.795s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 917.658818] env[62499]: INFO nova.compute.manager [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] [instance: cbe51201-c036-4f90-9372-8cc2d033f7a2] Took 1.03 seconds to deallocate network for instance. [ 917.750410] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.230s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 917.750942] env[62499]: DEBUG nova.compute.manager [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 917.753749] env[62499]: DEBUG oslo_concurrency.lockutils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.426s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 917.803183] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': session[5219b453-ddc7-8e28-3ba2-af7e13031566]52e119a1-3ada-b713-e2cb-d7f10e7ee82f, 'name': SearchDatastore_Task, 'duration_secs': 0.008496} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 917.803402] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Releasing lock "[datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 917.803658] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore2] b534fcdb-b0db-4e07-85fc-696dc58168aa/b534fcdb-b0db-4e07-85fc-696dc58168aa.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1423}} [ 917.803905] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-298f0d89-976b-44b8-9460-47486526b30e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 917.810316] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 917.810316] env[62499]: value = "task-1335781" [ 917.810316] env[62499]: _type = "Task" [ 917.810316] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 917.818012] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335781, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 917.950206] env[62499]: DEBUG nova.compute.manager [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 918.263856] env[62499]: DEBUG nova.compute.utils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 918.265371] env[62499]: DEBUG nova.compute.manager [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 918.265534] env[62499]: DEBUG nova.network.neutron [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 918.312391] env[62499]: DEBUG nova.policy [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '407c9b11e8f74200b1c4e34698925a92', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '616ea9f31e4b4823a1a50b89e3ea4c04', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 918.324758] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335781, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.454382} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 918.325109] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/0422123c-16ce-4c9d-b1c6-0281fe0e54ad/0422123c-16ce-4c9d-b1c6-0281fe0e54ad.vmdk to [datastore2] b534fcdb-b0db-4e07-85fc-696dc58168aa/b534fcdb-b0db-4e07-85fc-696dc58168aa.vmdk {{(pid=62499) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1434}} [ 918.325770] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Extending root virtual disk to 1048576 {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 918.325770] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-a87f9250-5954-4ac8-b4d7-7a1995646132 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.333924] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 918.333924] env[62499]: value = "task-1335782" [ 918.333924] env[62499]: _type = "Task" [ 918.333924] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 918.341695] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335782, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 918.412767] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cbf1452c-74fc-44a9-b9b2-2192d2dbcd86 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.420318] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3eab842-45d6-47fe-8f67-9fe6f2dcddc7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.451591] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb982eec-0717-4f30-a593-0d28fbdaa253 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.463990] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-48f4cdc3-424a-49a1-8a8a-3bd038561960 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.468841] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 918.483399] env[62499]: DEBUG nova.compute.provider_tree [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 918.688948] env[62499]: INFO nova.scheduler.client.report [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Deleted allocations for instance cbe51201-c036-4f90-9372-8cc2d033f7a2 [ 918.768115] env[62499]: DEBUG nova.compute.manager [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 918.826109] env[62499]: DEBUG nova.network.neutron [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Successfully created port: b2333580-9163-4b2c-bd7f-1a2225de00b4 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 918.843169] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335782, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.059918} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 918.843480] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Extended root virtual disk {{(pid=62499) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 918.844295] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4a962175-2cc2-46b8-8796-900086a4e283 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.866551] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Reconfiguring VM instance instance-0000004e to attach disk [datastore2] b534fcdb-b0db-4e07-85fc-696dc58168aa/b534fcdb-b0db-4e07-85fc-696dc58168aa.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 918.866910] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-c681725a-c94b-464c-b36c-a56c794651a3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.889702] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 918.889702] env[62499]: value = "task-1335783" [ 918.889702] env[62499]: _type = "Task" [ 918.889702] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 918.898412] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335783, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 918.986292] env[62499]: DEBUG nova.scheduler.client.report [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 919.198611] env[62499]: DEBUG oslo_concurrency.lockutils [None req-22b68b03-6b51-450c-905b-0fcd89def90e tempest-ServerActionsTestJSON-711130886 tempest-ServerActionsTestJSON-711130886-project-member] Lock "cbe51201-c036-4f90-9372-8cc2d033f7a2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 114.640s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 919.403794] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335783, 'name': ReconfigVM_Task, 'duration_secs': 0.285904} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 919.403794] env[62499]: DEBUG nova.virt.vmwareapi.volumeops [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Reconfigured VM instance instance-0000004e to attach disk [datastore2] b534fcdb-b0db-4e07-85fc-696dc58168aa/b534fcdb-b0db-4e07-85fc-696dc58168aa.vmdk or device None with type sparse {{(pid=62499) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 919.403794] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-e25521bc-de83-4227-81d1-44afe2d4a96f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.410723] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 919.410723] env[62499]: value = "task-1335784" [ 919.410723] env[62499]: _type = "Task" [ 919.410723] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 919.416879] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335784, 'name': Rename_Task} progress is 5%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 919.492735] env[62499]: DEBUG oslo_concurrency.lockutils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.738s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 919.492735] env[62499]: ERROR nova.compute.manager [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 91890880-66f0-40a4-a58f-6ad5b358a57a, please check neutron logs for more information. [ 919.492735] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Traceback (most recent call last): [ 919.492735] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 919.492735] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] self.driver.spawn(context, instance, image_meta, [ 919.492735] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 919.492735] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] self._vmops.spawn(context, instance, image_meta, injected_files, [ 919.492735] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 919.492735] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] vm_ref = self.build_virtual_machine(instance, [ 919.493049] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 919.493049] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] vif_infos = vmwarevif.get_vif_info(self._session, [ 919.493049] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 919.493049] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] for vif in network_info: [ 919.493049] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 919.493049] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] return self._sync_wrapper(fn, *args, **kwargs) [ 919.493049] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 919.493049] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] self.wait() [ 919.493049] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 919.493049] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] self[:] = self._gt.wait() [ 919.493049] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 919.493049] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] return self._exit_event.wait() [ 919.493049] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 919.493380] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] current.throw(*self._exc) [ 919.493380] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 919.493380] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] result = function(*args, **kwargs) [ 919.493380] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 919.493380] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] return func(*args, **kwargs) [ 919.493380] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 919.493380] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] raise e [ 919.493380] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 919.493380] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] nwinfo = self.network_api.allocate_for_instance( [ 919.493380] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 919.493380] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] created_port_ids = self._update_ports_for_instance( [ 919.493380] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 919.493380] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] with excutils.save_and_reraise_exception(): [ 919.493684] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 919.493684] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] self.force_reraise() [ 919.493684] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 919.493684] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] raise self.value [ 919.493684] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 919.493684] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] updated_port = self._update_port( [ 919.493684] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 919.493684] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] _ensure_no_port_binding_failure(port) [ 919.493684] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 919.493684] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] raise exception.PortBindingFailed(port_id=port['id']) [ 919.493684] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] nova.exception.PortBindingFailed: Binding failed for port 91890880-66f0-40a4-a58f-6ad5b358a57a, please check neutron logs for more information. [ 919.493684] env[62499]: ERROR nova.compute.manager [instance: a10b9328-394e-4673-928e-e6fa87b8c844] [ 919.493983] env[62499]: DEBUG nova.compute.utils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Binding failed for port 91890880-66f0-40a4-a58f-6ad5b358a57a, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 919.494924] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.576s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 919.499204] env[62499]: DEBUG nova.compute.manager [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Build of instance a10b9328-394e-4673-928e-e6fa87b8c844 was re-scheduled: Binding failed for port 91890880-66f0-40a4-a58f-6ad5b358a57a, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 919.499204] env[62499]: DEBUG nova.compute.manager [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 919.499204] env[62499]: DEBUG oslo_concurrency.lockutils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Acquiring lock "refresh_cache-a10b9328-394e-4673-928e-e6fa87b8c844" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 919.499204] env[62499]: DEBUG oslo_concurrency.lockutils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Acquired lock "refresh_cache-a10b9328-394e-4673-928e-e6fa87b8c844" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 919.499403] env[62499]: DEBUG nova.network.neutron [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 919.783363] env[62499]: DEBUG nova.compute.manager [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 919.795574] env[62499]: DEBUG nova.compute.manager [req-a8f70e7f-4e6c-476e-b4da-d6b2def7cb19 req-a35fdc0f-e794-4fc5-b578-edc0a50b7c65 service nova] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Received event network-changed-b2333580-9163-4b2c-bd7f-1a2225de00b4 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 919.795815] env[62499]: DEBUG nova.compute.manager [req-a8f70e7f-4e6c-476e-b4da-d6b2def7cb19 req-a35fdc0f-e794-4fc5-b578-edc0a50b7c65 service nova] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Refreshing instance network info cache due to event network-changed-b2333580-9163-4b2c-bd7f-1a2225de00b4. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 919.796044] env[62499]: DEBUG oslo_concurrency.lockutils [req-a8f70e7f-4e6c-476e-b4da-d6b2def7cb19 req-a35fdc0f-e794-4fc5-b578-edc0a50b7c65 service nova] Acquiring lock "refresh_cache-f1b5512b-702f-45b1-9844-13a1218c31b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 919.796181] env[62499]: DEBUG oslo_concurrency.lockutils [req-a8f70e7f-4e6c-476e-b4da-d6b2def7cb19 req-a35fdc0f-e794-4fc5-b578-edc0a50b7c65 service nova] Acquired lock "refresh_cache-f1b5512b-702f-45b1-9844-13a1218c31b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 919.796343] env[62499]: DEBUG nova.network.neutron [req-a8f70e7f-4e6c-476e-b4da-d6b2def7cb19 req-a35fdc0f-e794-4fc5-b578-edc0a50b7c65 service nova] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Refreshing network info cache for port b2333580-9163-4b2c-bd7f-1a2225de00b4 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 919.807711] env[62499]: DEBUG nova.virt.hardware [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 919.807898] env[62499]: DEBUG nova.virt.hardware [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 919.807996] env[62499]: DEBUG nova.virt.hardware [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 919.808116] env[62499]: DEBUG nova.virt.hardware [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 919.808284] env[62499]: DEBUG nova.virt.hardware [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 919.808402] env[62499]: DEBUG nova.virt.hardware [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 919.808604] env[62499]: DEBUG nova.virt.hardware [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 919.808762] env[62499]: DEBUG nova.virt.hardware [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 919.808922] env[62499]: DEBUG nova.virt.hardware [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 919.809103] env[62499]: DEBUG nova.virt.hardware [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 919.809283] env[62499]: DEBUG nova.virt.hardware [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 919.810363] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ee39969-2567-4812-b709-909326456651 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.819606] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2be7cb7-1ff2-4481-ba1c-0830a5bca331 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.924021] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335784, 'name': Rename_Task, 'duration_secs': 0.146672} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 919.924021] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Powering on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1452}} [ 919.924021] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-d90c1a13-b323-4c3a-8193-1a6e581f6847 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.931055] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 919.931055] env[62499]: value = "task-1335785" [ 919.931055] env[62499]: _type = "Task" [ 919.931055] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 919.940044] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335785, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 919.980205] env[62499]: ERROR nova.compute.manager [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b2333580-9163-4b2c-bd7f-1a2225de00b4, please check neutron logs for more information. [ 919.980205] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 919.980205] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 919.980205] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 919.980205] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 919.980205] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 919.980205] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 919.980205] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 919.980205] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 919.980205] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 919.980205] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 919.980205] env[62499]: ERROR nova.compute.manager raise self.value [ 919.980205] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 919.980205] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 919.980205] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 919.980205] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 919.980675] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 919.980675] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 919.980675] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b2333580-9163-4b2c-bd7f-1a2225de00b4, please check neutron logs for more information. [ 919.980675] env[62499]: ERROR nova.compute.manager [ 919.980675] env[62499]: Traceback (most recent call last): [ 919.980675] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 919.980675] env[62499]: listener.cb(fileno) [ 919.980675] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 919.980675] env[62499]: result = function(*args, **kwargs) [ 919.980675] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 919.980675] env[62499]: return func(*args, **kwargs) [ 919.980675] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 919.980675] env[62499]: raise e [ 919.980675] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 919.980675] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 919.980675] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 919.980675] env[62499]: created_port_ids = self._update_ports_for_instance( [ 919.980675] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 919.980675] env[62499]: with excutils.save_and_reraise_exception(): [ 919.980675] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 919.980675] env[62499]: self.force_reraise() [ 919.980675] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 919.980675] env[62499]: raise self.value [ 919.980675] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 919.980675] env[62499]: updated_port = self._update_port( [ 919.980675] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 919.980675] env[62499]: _ensure_no_port_binding_failure(port) [ 919.980675] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 919.980675] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 919.981662] env[62499]: nova.exception.PortBindingFailed: Binding failed for port b2333580-9163-4b2c-bd7f-1a2225de00b4, please check neutron logs for more information. [ 919.981662] env[62499]: Removing descriptor: 16 [ 919.981662] env[62499]: ERROR nova.compute.manager [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b2333580-9163-4b2c-bd7f-1a2225de00b4, please check neutron logs for more information. [ 919.981662] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Traceback (most recent call last): [ 919.981662] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 919.981662] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] yield resources [ 919.981662] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 919.981662] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] self.driver.spawn(context, instance, image_meta, [ 919.981662] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 919.981662] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 919.981662] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 919.981662] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] vm_ref = self.build_virtual_machine(instance, [ 919.982037] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 919.982037] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] vif_infos = vmwarevif.get_vif_info(self._session, [ 919.982037] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 919.982037] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] for vif in network_info: [ 919.982037] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 919.982037] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] return self._sync_wrapper(fn, *args, **kwargs) [ 919.982037] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 919.982037] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] self.wait() [ 919.982037] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 919.982037] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] self[:] = self._gt.wait() [ 919.982037] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 919.982037] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] return self._exit_event.wait() [ 919.982037] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 919.982417] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] result = hub.switch() [ 919.982417] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 919.982417] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] return self.greenlet.switch() [ 919.982417] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 919.982417] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] result = function(*args, **kwargs) [ 919.982417] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 919.982417] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] return func(*args, **kwargs) [ 919.982417] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 919.982417] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] raise e [ 919.982417] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 919.982417] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] nwinfo = self.network_api.allocate_for_instance( [ 919.982417] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 919.982417] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] created_port_ids = self._update_ports_for_instance( [ 919.982773] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 919.982773] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] with excutils.save_and_reraise_exception(): [ 919.982773] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 919.982773] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] self.force_reraise() [ 919.982773] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 919.982773] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] raise self.value [ 919.982773] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 919.982773] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] updated_port = self._update_port( [ 919.982773] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 919.982773] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] _ensure_no_port_binding_failure(port) [ 919.982773] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 919.982773] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] raise exception.PortBindingFailed(port_id=port['id']) [ 919.983122] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] nova.exception.PortBindingFailed: Binding failed for port b2333580-9163-4b2c-bd7f-1a2225de00b4, please check neutron logs for more information. [ 919.983122] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] [ 919.983122] env[62499]: INFO nova.compute.manager [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Terminating instance [ 919.984369] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Acquiring lock "refresh_cache-f1b5512b-702f-45b1-9844-13a1218c31b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 920.030308] env[62499]: DEBUG nova.network.neutron [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 920.128059] env[62499]: DEBUG nova.network.neutron [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 920.207736] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33278382-d5ae-404d-9c18-d7525cab5b43 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 920.217232] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc36ab9e-39b5-4478-abcb-2e2f3f2cb8da {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 920.252172] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb4b6365-76c0-49cf-88f9-27a997fcc886 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 920.257995] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-06a924ca-a61c-4cf2-9fdb-81bb5795c582 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 920.274676] env[62499]: DEBUG nova.compute.provider_tree [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 920.321509] env[62499]: DEBUG nova.network.neutron [req-a8f70e7f-4e6c-476e-b4da-d6b2def7cb19 req-a35fdc0f-e794-4fc5-b578-edc0a50b7c65 service nova] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 920.329030] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquiring lock "99920efd-13e6-4a18-9532-4b53bea3c736" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 920.329030] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "99920efd-13e6-4a18-9532-4b53bea3c736" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 920.427111] env[62499]: DEBUG nova.network.neutron [req-a8f70e7f-4e6c-476e-b4da-d6b2def7cb19 req-a35fdc0f-e794-4fc5-b578-edc0a50b7c65 service nova] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 920.443462] env[62499]: DEBUG oslo_vmware.api [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335785, 'name': PowerOnVM_Task, 'duration_secs': 0.499784} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 920.443680] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Powered on the VM {{(pid=62499) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1458}} [ 920.444011] env[62499]: DEBUG nova.compute.manager [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Checking state {{(pid=62499) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 920.444918] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e83630a6-8cc8-4496-a44b-447efbd06eab {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 920.633865] env[62499]: DEBUG oslo_concurrency.lockutils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Releasing lock "refresh_cache-a10b9328-394e-4673-928e-e6fa87b8c844" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 920.634137] env[62499]: DEBUG nova.compute.manager [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 920.634326] env[62499]: DEBUG nova.compute.manager [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 920.634488] env[62499]: DEBUG nova.network.neutron [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 920.650215] env[62499]: DEBUG nova.network.neutron [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 920.777043] env[62499]: DEBUG nova.scheduler.client.report [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 920.831081] env[62499]: DEBUG nova.compute.manager [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 920.929843] env[62499]: DEBUG oslo_concurrency.lockutils [req-a8f70e7f-4e6c-476e-b4da-d6b2def7cb19 req-a35fdc0f-e794-4fc5-b578-edc0a50b7c65 service nova] Releasing lock "refresh_cache-f1b5512b-702f-45b1-9844-13a1218c31b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 920.930350] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Acquired lock "refresh_cache-f1b5512b-702f-45b1-9844-13a1218c31b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 920.930542] env[62499]: DEBUG nova.network.neutron [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 920.960678] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 921.154304] env[62499]: DEBUG nova.network.neutron [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 921.283488] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.788s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 921.284175] env[62499]: ERROR nova.compute.manager [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port aa3bd8c0-68d7-4a18-86d7-24b006731bea, please check neutron logs for more information. [ 921.284175] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] Traceback (most recent call last): [ 921.284175] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 921.284175] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] self.driver.spawn(context, instance, image_meta, [ 921.284175] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 921.284175] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] self._vmops.spawn(context, instance, image_meta, injected_files, [ 921.284175] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 921.284175] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] vm_ref = self.build_virtual_machine(instance, [ 921.284175] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 921.284175] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] vif_infos = vmwarevif.get_vif_info(self._session, [ 921.284175] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 921.284981] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] for vif in network_info: [ 921.284981] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 921.284981] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] return self._sync_wrapper(fn, *args, **kwargs) [ 921.284981] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 921.284981] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] self.wait() [ 921.284981] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 921.284981] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] self[:] = self._gt.wait() [ 921.284981] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 921.284981] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] return self._exit_event.wait() [ 921.284981] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 921.284981] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] current.throw(*self._exc) [ 921.284981] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 921.284981] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] result = function(*args, **kwargs) [ 921.285843] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 921.285843] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] return func(*args, **kwargs) [ 921.285843] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 921.285843] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] raise e [ 921.285843] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 921.285843] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] nwinfo = self.network_api.allocate_for_instance( [ 921.285843] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 921.285843] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] created_port_ids = self._update_ports_for_instance( [ 921.285843] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 921.285843] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] with excutils.save_and_reraise_exception(): [ 921.285843] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 921.285843] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] self.force_reraise() [ 921.285843] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 921.286726] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] raise self.value [ 921.286726] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 921.286726] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] updated_port = self._update_port( [ 921.286726] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 921.286726] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] _ensure_no_port_binding_failure(port) [ 921.286726] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 921.286726] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] raise exception.PortBindingFailed(port_id=port['id']) [ 921.286726] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] nova.exception.PortBindingFailed: Binding failed for port aa3bd8c0-68d7-4a18-86d7-24b006731bea, please check neutron logs for more information. [ 921.286726] env[62499]: ERROR nova.compute.manager [instance: d308417b-47d0-423a-a603-56d9251ef818] [ 921.286726] env[62499]: DEBUG nova.compute.utils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Binding failed for port aa3bd8c0-68d7-4a18-86d7-24b006731bea, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 921.287818] env[62499]: DEBUG oslo_concurrency.lockutils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 9.852s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 921.287818] env[62499]: INFO nova.compute.claims [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 921.291307] env[62499]: DEBUG nova.compute.manager [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Build of instance d308417b-47d0-423a-a603-56d9251ef818 was re-scheduled: Binding failed for port aa3bd8c0-68d7-4a18-86d7-24b006731bea, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 921.291760] env[62499]: DEBUG nova.compute.manager [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 921.291987] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Acquiring lock "refresh_cache-d308417b-47d0-423a-a603-56d9251ef818" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 921.292147] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Acquired lock "refresh_cache-d308417b-47d0-423a-a603-56d9251ef818" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 921.292327] env[62499]: DEBUG nova.network.neutron [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 921.365588] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 921.461230] env[62499]: DEBUG nova.network.neutron [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 921.554091] env[62499]: DEBUG nova.network.neutron [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 921.655792] env[62499]: INFO nova.compute.manager [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] [instance: a10b9328-394e-4673-928e-e6fa87b8c844] Took 1.02 seconds to deallocate network for instance. [ 921.694082] env[62499]: DEBUG oslo_concurrency.lockutils [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquiring lock "b534fcdb-b0db-4e07-85fc-696dc58168aa" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 921.694349] env[62499]: DEBUG oslo_concurrency.lockutils [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Lock "b534fcdb-b0db-4e07-85fc-696dc58168aa" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 921.694664] env[62499]: DEBUG oslo_concurrency.lockutils [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquiring lock "b534fcdb-b0db-4e07-85fc-696dc58168aa-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 921.694765] env[62499]: DEBUG oslo_concurrency.lockutils [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Lock "b534fcdb-b0db-4e07-85fc-696dc58168aa-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 921.694886] env[62499]: DEBUG oslo_concurrency.lockutils [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Lock "b534fcdb-b0db-4e07-85fc-696dc58168aa-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 921.697153] env[62499]: INFO nova.compute.manager [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Terminating instance [ 921.698826] env[62499]: DEBUG oslo_concurrency.lockutils [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquiring lock "refresh_cache-b534fcdb-b0db-4e07-85fc-696dc58168aa" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 921.698979] env[62499]: DEBUG oslo_concurrency.lockutils [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquired lock "refresh_cache-b534fcdb-b0db-4e07-85fc-696dc58168aa" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 921.699183] env[62499]: DEBUG nova.network.neutron [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 921.821184] env[62499]: DEBUG nova.network.neutron [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 921.837880] env[62499]: DEBUG nova.compute.manager [req-70d56689-6d6a-4187-9c67-b2938d040711 req-fe5de2ff-00ad-496c-837c-5901a5c09e67 service nova] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Received event network-vif-deleted-b2333580-9163-4b2c-bd7f-1a2225de00b4 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 921.912484] env[62499]: DEBUG nova.network.neutron [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 922.056649] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Releasing lock "refresh_cache-f1b5512b-702f-45b1-9844-13a1218c31b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 922.057108] env[62499]: DEBUG nova.compute.manager [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 922.057304] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 922.057602] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5fa420cd-28c0-41cd-8120-9ce2f3ca0eff {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.067319] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5240627-585e-439c-b338-b46f800e2805 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.089884] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance f1b5512b-702f-45b1-9844-13a1218c31b3 could not be found. [ 922.090204] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 922.090412] env[62499]: INFO nova.compute.manager [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Took 0.03 seconds to destroy the instance on the hypervisor. [ 922.090658] env[62499]: DEBUG oslo.service.loopingcall [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 922.090878] env[62499]: DEBUG nova.compute.manager [-] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 922.090995] env[62499]: DEBUG nova.network.neutron [-] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 922.106270] env[62499]: DEBUG nova.network.neutron [-] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 922.215515] env[62499]: DEBUG nova.network.neutron [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 922.268836] env[62499]: DEBUG nova.network.neutron [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 922.414986] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Releasing lock "refresh_cache-d308417b-47d0-423a-a603-56d9251ef818" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 922.415244] env[62499]: DEBUG nova.compute.manager [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 922.415425] env[62499]: DEBUG nova.compute.manager [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 922.415592] env[62499]: DEBUG nova.network.neutron [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 922.438245] env[62499]: DEBUG nova.network.neutron [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 922.446797] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5641c97c-6566-448e-880b-5e64ad5d83b9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.455600] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d1993e6-f6c5-4d9e-9efb-ee3377fd075f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.485645] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b960193e-ed5d-498a-afd8-f684853444b6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.492922] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3ded711-a47b-47d6-9098-1ff0ac8cb264 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.507901] env[62499]: DEBUG nova.compute.provider_tree [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 922.608797] env[62499]: DEBUG nova.network.neutron [-] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 922.688759] env[62499]: INFO nova.scheduler.client.report [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Deleted allocations for instance a10b9328-394e-4673-928e-e6fa87b8c844 [ 922.771197] env[62499]: DEBUG oslo_concurrency.lockutils [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Releasing lock "refresh_cache-b534fcdb-b0db-4e07-85fc-696dc58168aa" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 922.771592] env[62499]: DEBUG nova.compute.manager [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 922.771784] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 922.772662] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe4fea86-080c-406f-a185-a2a7eebaca30 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.780780] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Powering off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1512}} [ 922.781010] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-c4b27296-8d49-4285-8765-47c9d589755c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.787023] env[62499]: DEBUG oslo_vmware.api [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 922.787023] env[62499]: value = "task-1335786" [ 922.787023] env[62499]: _type = "Task" [ 922.787023] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 922.794719] env[62499]: DEBUG oslo_vmware.api [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335786, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 922.942056] env[62499]: DEBUG nova.network.neutron [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 923.011142] env[62499]: DEBUG nova.scheduler.client.report [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 923.111283] env[62499]: INFO nova.compute.manager [-] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Took 1.02 seconds to deallocate network for instance. [ 923.113605] env[62499]: DEBUG nova.compute.claims [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 923.113777] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 923.196892] env[62499]: DEBUG oslo_concurrency.lockutils [None req-610ab5d6-e011-4a99-af58-edb9cdf6804b tempest-ServerAddressesNegativeTestJSON-1867545015 tempest-ServerAddressesNegativeTestJSON-1867545015-project-member] Lock "a10b9328-394e-4673-928e-e6fa87b8c844" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 97.574s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 923.297775] env[62499]: DEBUG oslo_vmware.api [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335786, 'name': PowerOffVM_Task, 'duration_secs': 0.19458} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 923.298655] env[62499]: DEBUG nova.virt.vmwareapi.vm_util [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Powered off the VM {{(pid=62499) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1517}} [ 923.298655] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Unregistering the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1093}} [ 923.298655] env[62499]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-1e150816-c3d5-4b9f-b054-bb28e01a68e1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 923.321699] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Unregistered the VM {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1096}} [ 923.321931] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Deleting contents of the VM from datastore datastore2 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1106}} [ 923.322135] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Deleting the datastore file [datastore2] b534fcdb-b0db-4e07-85fc-696dc58168aa {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 923.322480] env[62499]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-d3822bb5-6715-4e93-8973-f0240c337b2c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 923.329239] env[62499]: DEBUG oslo_vmware.api [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for the task: (returnval){ [ 923.329239] env[62499]: value = "task-1335788" [ 923.329239] env[62499]: _type = "Task" [ 923.329239] env[62499]: } to complete. {{(pid=62499) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 923.336930] env[62499]: DEBUG oslo_vmware.api [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335788, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 923.444879] env[62499]: INFO nova.compute.manager [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] [instance: d308417b-47d0-423a-a603-56d9251ef818] Took 1.03 seconds to deallocate network for instance. [ 923.516050] env[62499]: DEBUG oslo_concurrency.lockutils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.230s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 923.516446] env[62499]: DEBUG nova.compute.manager [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 923.518999] env[62499]: DEBUG oslo_concurrency.lockutils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 10.348s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 923.520397] env[62499]: INFO nova.compute.claims [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 923.838554] env[62499]: DEBUG oslo_vmware.api [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Task: {'id': task-1335788, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.103073} completed successfully. {{(pid=62499) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 923.838794] env[62499]: DEBUG nova.virt.vmwareapi.ds_util [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Deleted the datastore file {{(pid=62499) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 923.839057] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Deleted contents of the VM from datastore datastore2 {{(pid=62499) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1116}} [ 923.839158] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 923.839329] env[62499]: INFO nova.compute.manager [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Took 1.07 seconds to destroy the instance on the hypervisor. [ 923.839568] env[62499]: DEBUG oslo.service.loopingcall [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 923.839754] env[62499]: DEBUG nova.compute.manager [-] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 923.840015] env[62499]: DEBUG nova.network.neutron [-] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 923.856944] env[62499]: DEBUG nova.network.neutron [-] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 924.025016] env[62499]: DEBUG nova.compute.utils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 924.032213] env[62499]: DEBUG nova.compute.manager [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 924.032401] env[62499]: DEBUG nova.network.neutron [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 924.078298] env[62499]: DEBUG nova.policy [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c0026a72212047e48515d4e1d7c43f9b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7e4a0adf95ad42cfa20c49eda388b6ce', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 924.327978] env[62499]: DEBUG nova.network.neutron [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Successfully created port: 21aa756e-d6a2-47b0-92ea-2b12642e43a1 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 924.359746] env[62499]: DEBUG nova.network.neutron [-] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 924.497367] env[62499]: INFO nova.scheduler.client.report [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Deleted allocations for instance d308417b-47d0-423a-a603-56d9251ef818 [ 924.533625] env[62499]: DEBUG nova.compute.manager [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 924.700318] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6f3ae69-07f9-4ba6-b73b-952e2107187a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 924.709385] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ac80327-b835-4544-8578-677870b4c0ac {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 924.742273] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bac272de-e094-4799-b372-d85a4a81bf4b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 924.749770] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60cd0272-9b02-4acb-b0b9-2607151fe1f9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 924.763948] env[62499]: DEBUG nova.compute.provider_tree [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 924.863325] env[62499]: INFO nova.compute.manager [-] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Took 1.02 seconds to deallocate network for instance. [ 925.010566] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2cfec637-10c0-4507-8b23-d16501f8ae53 tempest-ServerAddressesTestJSON-249145922 tempest-ServerAddressesTestJSON-249145922-project-member] Lock "d308417b-47d0-423a-a603-56d9251ef818" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 91.484s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 925.079878] env[62499]: DEBUG nova.compute.manager [req-31ddf0f7-eea3-4829-aad7-322270d32ed3 req-82a167ff-a93a-4c2c-9be1-5ab2a32ee0e6 service nova] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Received event network-changed-21aa756e-d6a2-47b0-92ea-2b12642e43a1 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 925.079878] env[62499]: DEBUG nova.compute.manager [req-31ddf0f7-eea3-4829-aad7-322270d32ed3 req-82a167ff-a93a-4c2c-9be1-5ab2a32ee0e6 service nova] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Refreshing instance network info cache due to event network-changed-21aa756e-d6a2-47b0-92ea-2b12642e43a1. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 925.081732] env[62499]: DEBUG oslo_concurrency.lockutils [req-31ddf0f7-eea3-4829-aad7-322270d32ed3 req-82a167ff-a93a-4c2c-9be1-5ab2a32ee0e6 service nova] Acquiring lock "refresh_cache-4af6c7f4-b774-4c3f-80af-e8704a904c38" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 925.081732] env[62499]: DEBUG oslo_concurrency.lockutils [req-31ddf0f7-eea3-4829-aad7-322270d32ed3 req-82a167ff-a93a-4c2c-9be1-5ab2a32ee0e6 service nova] Acquired lock "refresh_cache-4af6c7f4-b774-4c3f-80af-e8704a904c38" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 925.081732] env[62499]: DEBUG nova.network.neutron [req-31ddf0f7-eea3-4829-aad7-322270d32ed3 req-82a167ff-a93a-4c2c-9be1-5ab2a32ee0e6 service nova] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Refreshing network info cache for port 21aa756e-d6a2-47b0-92ea-2b12642e43a1 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 925.249931] env[62499]: ERROR nova.compute.manager [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 21aa756e-d6a2-47b0-92ea-2b12642e43a1, please check neutron logs for more information. [ 925.249931] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 925.249931] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 925.249931] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 925.249931] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 925.249931] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 925.249931] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 925.249931] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 925.249931] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 925.249931] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 925.249931] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 925.249931] env[62499]: ERROR nova.compute.manager raise self.value [ 925.249931] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 925.249931] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 925.249931] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 925.249931] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 925.250504] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 925.250504] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 925.250504] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 21aa756e-d6a2-47b0-92ea-2b12642e43a1, please check neutron logs for more information. [ 925.250504] env[62499]: ERROR nova.compute.manager [ 925.250504] env[62499]: Traceback (most recent call last): [ 925.250504] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 925.250504] env[62499]: listener.cb(fileno) [ 925.250504] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 925.250504] env[62499]: result = function(*args, **kwargs) [ 925.250504] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 925.250504] env[62499]: return func(*args, **kwargs) [ 925.250504] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 925.250504] env[62499]: raise e [ 925.250504] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 925.250504] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 925.250504] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 925.250504] env[62499]: created_port_ids = self._update_ports_for_instance( [ 925.250504] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 925.250504] env[62499]: with excutils.save_and_reraise_exception(): [ 925.250504] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 925.250504] env[62499]: self.force_reraise() [ 925.250504] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 925.250504] env[62499]: raise self.value [ 925.250504] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 925.250504] env[62499]: updated_port = self._update_port( [ 925.250504] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 925.250504] env[62499]: _ensure_no_port_binding_failure(port) [ 925.250504] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 925.250504] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 925.251376] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 21aa756e-d6a2-47b0-92ea-2b12642e43a1, please check neutron logs for more information. [ 925.251376] env[62499]: Removing descriptor: 16 [ 925.270375] env[62499]: DEBUG nova.scheduler.client.report [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 925.369918] env[62499]: DEBUG oslo_concurrency.lockutils [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 925.548296] env[62499]: DEBUG nova.compute.manager [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 925.574605] env[62499]: DEBUG nova.virt.hardware [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 925.574846] env[62499]: DEBUG nova.virt.hardware [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 925.575010] env[62499]: DEBUG nova.virt.hardware [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 925.575211] env[62499]: DEBUG nova.virt.hardware [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 925.575357] env[62499]: DEBUG nova.virt.hardware [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 925.575501] env[62499]: DEBUG nova.virt.hardware [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 925.575704] env[62499]: DEBUG nova.virt.hardware [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 925.575862] env[62499]: DEBUG nova.virt.hardware [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 925.576123] env[62499]: DEBUG nova.virt.hardware [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 925.576316] env[62499]: DEBUG nova.virt.hardware [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 925.576490] env[62499]: DEBUG nova.virt.hardware [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 925.577357] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2ae0908-6759-4176-963a-b44eee7972df {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 925.585156] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8ecbfec-10e5-48e3-9ff6-779aac394231 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 925.600470] env[62499]: ERROR nova.compute.manager [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 21aa756e-d6a2-47b0-92ea-2b12642e43a1, please check neutron logs for more information. [ 925.600470] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Traceback (most recent call last): [ 925.600470] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 925.600470] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] yield resources [ 925.600470] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 925.600470] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] self.driver.spawn(context, instance, image_meta, [ 925.600470] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 925.600470] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] self._vmops.spawn(context, instance, image_meta, injected_files, [ 925.600470] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 925.600470] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] vm_ref = self.build_virtual_machine(instance, [ 925.600470] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 925.601082] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] vif_infos = vmwarevif.get_vif_info(self._session, [ 925.601082] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 925.601082] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] for vif in network_info: [ 925.601082] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 925.601082] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] return self._sync_wrapper(fn, *args, **kwargs) [ 925.601082] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 925.601082] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] self.wait() [ 925.601082] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 925.601082] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] self[:] = self._gt.wait() [ 925.601082] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 925.601082] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] return self._exit_event.wait() [ 925.601082] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 925.601082] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] current.throw(*self._exc) [ 925.601526] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 925.601526] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] result = function(*args, **kwargs) [ 925.601526] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 925.601526] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] return func(*args, **kwargs) [ 925.601526] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 925.601526] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] raise e [ 925.601526] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 925.601526] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] nwinfo = self.network_api.allocate_for_instance( [ 925.601526] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 925.601526] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] created_port_ids = self._update_ports_for_instance( [ 925.601526] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 925.601526] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] with excutils.save_and_reraise_exception(): [ 925.601526] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 925.602070] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] self.force_reraise() [ 925.602070] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 925.602070] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] raise self.value [ 925.602070] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 925.602070] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] updated_port = self._update_port( [ 925.602070] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 925.602070] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] _ensure_no_port_binding_failure(port) [ 925.602070] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 925.602070] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] raise exception.PortBindingFailed(port_id=port['id']) [ 925.602070] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] nova.exception.PortBindingFailed: Binding failed for port 21aa756e-d6a2-47b0-92ea-2b12642e43a1, please check neutron logs for more information. [ 925.602070] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] [ 925.602070] env[62499]: INFO nova.compute.manager [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Terminating instance [ 925.602732] env[62499]: DEBUG oslo_concurrency.lockutils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Acquiring lock "refresh_cache-4af6c7f4-b774-4c3f-80af-e8704a904c38" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 925.603476] env[62499]: DEBUG nova.network.neutron [req-31ddf0f7-eea3-4829-aad7-322270d32ed3 req-82a167ff-a93a-4c2c-9be1-5ab2a32ee0e6 service nova] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 925.691374] env[62499]: DEBUG nova.network.neutron [req-31ddf0f7-eea3-4829-aad7-322270d32ed3 req-82a167ff-a93a-4c2c-9be1-5ab2a32ee0e6 service nova] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 925.775314] env[62499]: DEBUG oslo_concurrency.lockutils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.256s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 925.776245] env[62499]: DEBUG nova.compute.manager [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 925.779061] env[62499]: DEBUG oslo_concurrency.lockutils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 11.879s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 925.781920] env[62499]: INFO nova.compute.claims [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 926.193521] env[62499]: DEBUG oslo_concurrency.lockutils [req-31ddf0f7-eea3-4829-aad7-322270d32ed3 req-82a167ff-a93a-4c2c-9be1-5ab2a32ee0e6 service nova] Releasing lock "refresh_cache-4af6c7f4-b774-4c3f-80af-e8704a904c38" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 926.194261] env[62499]: DEBUG oslo_concurrency.lockutils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Acquired lock "refresh_cache-4af6c7f4-b774-4c3f-80af-e8704a904c38" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 926.194634] env[62499]: DEBUG nova.network.neutron [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 926.286160] env[62499]: DEBUG nova.compute.utils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 926.289990] env[62499]: DEBUG nova.compute.manager [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 926.291033] env[62499]: DEBUG nova.network.neutron [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 926.343863] env[62499]: DEBUG nova.policy [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4762545dc5e14c16b5cbb1bad27d90a2', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3f11318c300c4c308f3f8a14db355954', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 926.784064] env[62499]: DEBUG nova.network.neutron [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 926.789238] env[62499]: DEBUG nova.network.neutron [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Successfully created port: d7f7e92e-10d8-46bd-8a94-a1d88ba5bbdd {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 926.791256] env[62499]: DEBUG nova.compute.manager [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 926.912801] env[62499]: DEBUG nova.network.neutron [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 926.985745] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2341dd63-cee4-447b-be96-7d143c2b9699 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 926.995086] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2c7adaf-a8f0-472b-a24c-c3baf79f183b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 927.032564] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2511a53-7a73-455b-9d18-071d2fcd3854 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 927.040145] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8afccb79-2079-4155-a2d3-f14202627d96 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 927.053468] env[62499]: DEBUG nova.compute.provider_tree [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 927.150116] env[62499]: DEBUG nova.compute.manager [req-ee7b483b-c62b-4550-8063-fdeac9021c8b req-9ce2b0cb-4af8-4a91-9ec4-7200a53cceaf service nova] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Received event network-vif-deleted-21aa756e-d6a2-47b0-92ea-2b12642e43a1 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 927.419017] env[62499]: DEBUG oslo_concurrency.lockutils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Releasing lock "refresh_cache-4af6c7f4-b774-4c3f-80af-e8704a904c38" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 927.419017] env[62499]: DEBUG nova.compute.manager [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 927.419017] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 927.419017] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-40e0a3f3-a30a-4940-a7f6-73e1ac7b0564 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 927.426771] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b8adc1a4-bf34-4dcb-8b05-2bd0f9103394 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 927.451694] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4af6c7f4-b774-4c3f-80af-e8704a904c38 could not be found. [ 927.451694] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 927.451694] env[62499]: INFO nova.compute.manager [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Took 0.03 seconds to destroy the instance on the hypervisor. [ 927.451694] env[62499]: DEBUG oslo.service.loopingcall [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 927.451694] env[62499]: DEBUG nova.compute.manager [-] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 927.451694] env[62499]: DEBUG nova.network.neutron [-] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 927.477825] env[62499]: DEBUG nova.network.neutron [-] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 927.559277] env[62499]: DEBUG nova.scheduler.client.report [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 927.807445] env[62499]: DEBUG nova.compute.manager [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 927.844288] env[62499]: DEBUG nova.virt.hardware [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 927.844288] env[62499]: DEBUG nova.virt.hardware [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 927.844288] env[62499]: DEBUG nova.virt.hardware [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 927.844288] env[62499]: DEBUG nova.virt.hardware [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 927.844476] env[62499]: DEBUG nova.virt.hardware [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 927.844476] env[62499]: DEBUG nova.virt.hardware [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 927.844476] env[62499]: DEBUG nova.virt.hardware [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 927.844744] env[62499]: DEBUG nova.virt.hardware [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 927.845101] env[62499]: DEBUG nova.virt.hardware [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 927.845424] env[62499]: DEBUG nova.virt.hardware [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 927.845746] env[62499]: DEBUG nova.virt.hardware [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 927.846882] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b41d8dc-2adf-4124-aa92-8e320e41922d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 927.857034] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d69a8b4b-d195-4930-bc69-58f1774a9f18 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 927.979595] env[62499]: DEBUG nova.network.neutron [-] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 928.062231] env[62499]: DEBUG oslo_concurrency.lockutils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.283s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 928.062759] env[62499]: DEBUG nova.compute.manager [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 928.065334] env[62499]: DEBUG oslo_concurrency.lockutils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.104s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 928.066803] env[62499]: INFO nova.compute.claims [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 928.485924] env[62499]: INFO nova.compute.manager [-] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Took 1.04 seconds to deallocate network for instance. [ 928.488477] env[62499]: DEBUG nova.compute.claims [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 928.488657] env[62499]: DEBUG oslo_concurrency.lockutils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 928.497560] env[62499]: ERROR nova.compute.manager [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d7f7e92e-10d8-46bd-8a94-a1d88ba5bbdd, please check neutron logs for more information. [ 928.497560] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 928.497560] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 928.497560] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 928.497560] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 928.497560] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 928.497560] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 928.497560] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 928.497560] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 928.497560] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 928.497560] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 928.497560] env[62499]: ERROR nova.compute.manager raise self.value [ 928.497560] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 928.497560] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 928.497560] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 928.497560] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 928.498241] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 928.498241] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 928.498241] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d7f7e92e-10d8-46bd-8a94-a1d88ba5bbdd, please check neutron logs for more information. [ 928.498241] env[62499]: ERROR nova.compute.manager [ 928.498241] env[62499]: Traceback (most recent call last): [ 928.498241] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 928.498241] env[62499]: listener.cb(fileno) [ 928.498241] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 928.498241] env[62499]: result = function(*args, **kwargs) [ 928.498241] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 928.498241] env[62499]: return func(*args, **kwargs) [ 928.498241] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 928.498241] env[62499]: raise e [ 928.498241] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 928.498241] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 928.498241] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 928.498241] env[62499]: created_port_ids = self._update_ports_for_instance( [ 928.498241] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 928.498241] env[62499]: with excutils.save_and_reraise_exception(): [ 928.498241] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 928.498241] env[62499]: self.force_reraise() [ 928.498241] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 928.498241] env[62499]: raise self.value [ 928.498241] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 928.498241] env[62499]: updated_port = self._update_port( [ 928.498241] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 928.498241] env[62499]: _ensure_no_port_binding_failure(port) [ 928.498241] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 928.498241] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 928.499117] env[62499]: nova.exception.PortBindingFailed: Binding failed for port d7f7e92e-10d8-46bd-8a94-a1d88ba5bbdd, please check neutron logs for more information. [ 928.499117] env[62499]: Removing descriptor: 16 [ 928.499117] env[62499]: ERROR nova.compute.manager [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d7f7e92e-10d8-46bd-8a94-a1d88ba5bbdd, please check neutron logs for more information. [ 928.499117] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Traceback (most recent call last): [ 928.499117] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 928.499117] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] yield resources [ 928.499117] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 928.499117] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] self.driver.spawn(context, instance, image_meta, [ 928.499117] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 928.499117] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 928.499117] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 928.499117] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] vm_ref = self.build_virtual_machine(instance, [ 928.499533] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 928.499533] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] vif_infos = vmwarevif.get_vif_info(self._session, [ 928.499533] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 928.499533] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] for vif in network_info: [ 928.499533] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 928.499533] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] return self._sync_wrapper(fn, *args, **kwargs) [ 928.499533] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 928.499533] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] self.wait() [ 928.499533] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 928.499533] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] self[:] = self._gt.wait() [ 928.499533] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 928.499533] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] return self._exit_event.wait() [ 928.499533] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 928.499934] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] result = hub.switch() [ 928.499934] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 928.499934] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] return self.greenlet.switch() [ 928.499934] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 928.499934] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] result = function(*args, **kwargs) [ 928.499934] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 928.499934] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] return func(*args, **kwargs) [ 928.499934] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 928.499934] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] raise e [ 928.499934] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 928.499934] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] nwinfo = self.network_api.allocate_for_instance( [ 928.499934] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 928.499934] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] created_port_ids = self._update_ports_for_instance( [ 928.500427] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 928.500427] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] with excutils.save_and_reraise_exception(): [ 928.500427] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 928.500427] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] self.force_reraise() [ 928.500427] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 928.500427] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] raise self.value [ 928.500427] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 928.500427] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] updated_port = self._update_port( [ 928.500427] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 928.500427] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] _ensure_no_port_binding_failure(port) [ 928.500427] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 928.500427] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] raise exception.PortBindingFailed(port_id=port['id']) [ 928.500881] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] nova.exception.PortBindingFailed: Binding failed for port d7f7e92e-10d8-46bd-8a94-a1d88ba5bbdd, please check neutron logs for more information. [ 928.500881] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] [ 928.500881] env[62499]: INFO nova.compute.manager [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Terminating instance [ 928.501733] env[62499]: DEBUG oslo_concurrency.lockutils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "refresh_cache-4fa63411-a721-4e34-a08d-90587b0b34d3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 928.501890] env[62499]: DEBUG oslo_concurrency.lockutils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquired lock "refresh_cache-4fa63411-a721-4e34-a08d-90587b0b34d3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 928.502069] env[62499]: DEBUG nova.network.neutron [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 928.571978] env[62499]: DEBUG nova.compute.utils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 928.575267] env[62499]: DEBUG nova.compute.manager [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 928.575394] env[62499]: DEBUG nova.network.neutron [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 928.641357] env[62499]: DEBUG nova.policy [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f4fd324459ba49f7b40aad54e650d925', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'fcf8aa231c564984b42197ad6a93e018', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 928.995034] env[62499]: DEBUG nova.network.neutron [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Successfully created port: 17a49625-1200-4a66-81d3-8a6e144d9234 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 929.029803] env[62499]: DEBUG nova.network.neutron [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 929.076725] env[62499]: DEBUG nova.compute.manager [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 929.186041] env[62499]: DEBUG nova.compute.manager [req-60c348c1-a491-4fcd-935c-0e735c621f67 req-f60b0925-038f-4a33-a4a2-abc7ca864035 service nova] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Received event network-changed-d7f7e92e-10d8-46bd-8a94-a1d88ba5bbdd {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 929.186254] env[62499]: DEBUG nova.compute.manager [req-60c348c1-a491-4fcd-935c-0e735c621f67 req-f60b0925-038f-4a33-a4a2-abc7ca864035 service nova] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Refreshing instance network info cache due to event network-changed-d7f7e92e-10d8-46bd-8a94-a1d88ba5bbdd. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 929.186448] env[62499]: DEBUG oslo_concurrency.lockutils [req-60c348c1-a491-4fcd-935c-0e735c621f67 req-f60b0925-038f-4a33-a4a2-abc7ca864035 service nova] Acquiring lock "refresh_cache-4fa63411-a721-4e34-a08d-90587b0b34d3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 929.199373] env[62499]: DEBUG nova.network.neutron [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 929.255017] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fdbd52e0-721c-45a5-9dce-cd4675847e9d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 929.262237] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77151c19-9038-4881-8f9f-754acb30478b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 929.295689] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8c613e4-20cb-4b5c-9b77-0673c98d1814 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 929.303370] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1e28c5b3-fc2f-4eee-b509-f3d43a271a25 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 929.318527] env[62499]: DEBUG nova.compute.provider_tree [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 929.459748] env[62499]: DEBUG oslo_concurrency.lockutils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Acquiring lock "4170a2f4-73ff-4bc6-9dc6-d2c4765c449d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 929.459748] env[62499]: DEBUG oslo_concurrency.lockutils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Lock "4170a2f4-73ff-4bc6-9dc6-d2c4765c449d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 929.704289] env[62499]: DEBUG oslo_concurrency.lockutils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Releasing lock "refresh_cache-4fa63411-a721-4e34-a08d-90587b0b34d3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 929.704707] env[62499]: DEBUG nova.compute.manager [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 929.704916] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 929.705263] env[62499]: DEBUG oslo_concurrency.lockutils [req-60c348c1-a491-4fcd-935c-0e735c621f67 req-f60b0925-038f-4a33-a4a2-abc7ca864035 service nova] Acquired lock "refresh_cache-4fa63411-a721-4e34-a08d-90587b0b34d3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 929.705431] env[62499]: DEBUG nova.network.neutron [req-60c348c1-a491-4fcd-935c-0e735c621f67 req-f60b0925-038f-4a33-a4a2-abc7ca864035 service nova] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Refreshing network info cache for port d7f7e92e-10d8-46bd-8a94-a1d88ba5bbdd {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 929.706483] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-098c3052-35ef-4edc-84e1-a0b6a1803a38 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 929.717980] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-63fcf4e7-6d0b-45d0-97ed-dbc5b4f4a6d8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 929.745529] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4fa63411-a721-4e34-a08d-90587b0b34d3 could not be found. [ 929.745756] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 929.746191] env[62499]: INFO nova.compute.manager [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Took 0.04 seconds to destroy the instance on the hypervisor. [ 929.746473] env[62499]: DEBUG oslo.service.loopingcall [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 929.746690] env[62499]: DEBUG nova.compute.manager [-] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 929.746788] env[62499]: DEBUG nova.network.neutron [-] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 929.765945] env[62499]: DEBUG nova.network.neutron [-] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 929.822487] env[62499]: DEBUG nova.scheduler.client.report [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 929.891301] env[62499]: ERROR nova.compute.manager [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 17a49625-1200-4a66-81d3-8a6e144d9234, please check neutron logs for more information. [ 929.891301] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 929.891301] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 929.891301] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 929.891301] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 929.891301] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 929.891301] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 929.891301] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 929.891301] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 929.891301] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 929.891301] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 929.891301] env[62499]: ERROR nova.compute.manager raise self.value [ 929.891301] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 929.891301] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 929.891301] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 929.891301] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 929.891824] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 929.891824] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 929.891824] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 17a49625-1200-4a66-81d3-8a6e144d9234, please check neutron logs for more information. [ 929.891824] env[62499]: ERROR nova.compute.manager [ 929.891824] env[62499]: Traceback (most recent call last): [ 929.891824] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 929.891824] env[62499]: listener.cb(fileno) [ 929.891824] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 929.891824] env[62499]: result = function(*args, **kwargs) [ 929.891824] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 929.891824] env[62499]: return func(*args, **kwargs) [ 929.891824] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 929.891824] env[62499]: raise e [ 929.891824] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 929.891824] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 929.891824] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 929.891824] env[62499]: created_port_ids = self._update_ports_for_instance( [ 929.891824] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 929.891824] env[62499]: with excutils.save_and_reraise_exception(): [ 929.891824] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 929.891824] env[62499]: self.force_reraise() [ 929.891824] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 929.891824] env[62499]: raise self.value [ 929.891824] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 929.891824] env[62499]: updated_port = self._update_port( [ 929.891824] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 929.891824] env[62499]: _ensure_no_port_binding_failure(port) [ 929.891824] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 929.891824] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 929.892675] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 17a49625-1200-4a66-81d3-8a6e144d9234, please check neutron logs for more information. [ 929.892675] env[62499]: Removing descriptor: 16 [ 929.961856] env[62499]: DEBUG nova.compute.manager [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 930.097059] env[62499]: DEBUG nova.compute.manager [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 930.133718] env[62499]: DEBUG nova.virt.hardware [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 930.133985] env[62499]: DEBUG nova.virt.hardware [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 930.134363] env[62499]: DEBUG nova.virt.hardware [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 930.134530] env[62499]: DEBUG nova.virt.hardware [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 930.134760] env[62499]: DEBUG nova.virt.hardware [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 930.134933] env[62499]: DEBUG nova.virt.hardware [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 930.135210] env[62499]: DEBUG nova.virt.hardware [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 930.135421] env[62499]: DEBUG nova.virt.hardware [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 930.135883] env[62499]: DEBUG nova.virt.hardware [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 930.136135] env[62499]: DEBUG nova.virt.hardware [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 930.136357] env[62499]: DEBUG nova.virt.hardware [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 930.137525] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-36e73290-41ff-4149-962f-61a12bf03ade {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 930.149262] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2430f89-9e14-4080-b76f-916bd4677e63 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 930.164159] env[62499]: ERROR nova.compute.manager [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 17a49625-1200-4a66-81d3-8a6e144d9234, please check neutron logs for more information. [ 930.164159] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] Traceback (most recent call last): [ 930.164159] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 930.164159] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] yield resources [ 930.164159] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 930.164159] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] self.driver.spawn(context, instance, image_meta, [ 930.164159] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 930.164159] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] self._vmops.spawn(context, instance, image_meta, injected_files, [ 930.164159] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 930.164159] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] vm_ref = self.build_virtual_machine(instance, [ 930.164159] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 930.164670] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] vif_infos = vmwarevif.get_vif_info(self._session, [ 930.164670] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 930.164670] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] for vif in network_info: [ 930.164670] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 930.164670] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] return self._sync_wrapper(fn, *args, **kwargs) [ 930.164670] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 930.164670] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] self.wait() [ 930.164670] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 930.164670] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] self[:] = self._gt.wait() [ 930.164670] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 930.164670] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] return self._exit_event.wait() [ 930.164670] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 930.164670] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] current.throw(*self._exc) [ 930.165018] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 930.165018] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] result = function(*args, **kwargs) [ 930.165018] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 930.165018] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] return func(*args, **kwargs) [ 930.165018] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 930.165018] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] raise e [ 930.165018] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 930.165018] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] nwinfo = self.network_api.allocate_for_instance( [ 930.165018] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 930.165018] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] created_port_ids = self._update_ports_for_instance( [ 930.165018] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 930.165018] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] with excutils.save_and_reraise_exception(): [ 930.165018] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 930.165360] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] self.force_reraise() [ 930.165360] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 930.165360] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] raise self.value [ 930.165360] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 930.165360] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] updated_port = self._update_port( [ 930.165360] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 930.165360] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] _ensure_no_port_binding_failure(port) [ 930.165360] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 930.165360] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] raise exception.PortBindingFailed(port_id=port['id']) [ 930.165360] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] nova.exception.PortBindingFailed: Binding failed for port 17a49625-1200-4a66-81d3-8a6e144d9234, please check neutron logs for more information. [ 930.165360] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] [ 930.165360] env[62499]: INFO nova.compute.manager [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Terminating instance [ 930.166894] env[62499]: DEBUG oslo_concurrency.lockutils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Acquiring lock "refresh_cache-88466440-7392-43d3-83ec-45229516a3ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 930.167012] env[62499]: DEBUG oslo_concurrency.lockutils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Acquired lock "refresh_cache-88466440-7392-43d3-83ec-45229516a3ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 930.167313] env[62499]: DEBUG nova.network.neutron [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 930.227240] env[62499]: DEBUG nova.network.neutron [req-60c348c1-a491-4fcd-935c-0e735c621f67 req-f60b0925-038f-4a33-a4a2-abc7ca864035 service nova] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 930.268008] env[62499]: DEBUG nova.network.neutron [-] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 930.305420] env[62499]: DEBUG nova.network.neutron [req-60c348c1-a491-4fcd-935c-0e735c621f67 req-f60b0925-038f-4a33-a4a2-abc7ca864035 service nova] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 930.329393] env[62499]: DEBUG oslo_concurrency.lockutils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.264s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 930.329956] env[62499]: DEBUG nova.compute.manager [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 930.332727] env[62499]: DEBUG oslo_concurrency.lockutils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.549s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 930.341247] env[62499]: INFO nova.compute.claims [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 930.488653] env[62499]: DEBUG oslo_concurrency.lockutils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 930.686905] env[62499]: DEBUG nova.network.neutron [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 930.772749] env[62499]: INFO nova.compute.manager [-] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Took 1.02 seconds to deallocate network for instance. [ 930.774622] env[62499]: DEBUG nova.compute.claims [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 930.774833] env[62499]: DEBUG oslo_concurrency.lockutils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 930.786784] env[62499]: DEBUG nova.network.neutron [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 930.805578] env[62499]: DEBUG oslo_concurrency.lockutils [req-60c348c1-a491-4fcd-935c-0e735c621f67 req-f60b0925-038f-4a33-a4a2-abc7ca864035 service nova] Releasing lock "refresh_cache-4fa63411-a721-4e34-a08d-90587b0b34d3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 930.805794] env[62499]: DEBUG nova.compute.manager [req-60c348c1-a491-4fcd-935c-0e735c621f67 req-f60b0925-038f-4a33-a4a2-abc7ca864035 service nova] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Received event network-vif-deleted-d7f7e92e-10d8-46bd-8a94-a1d88ba5bbdd {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 930.847869] env[62499]: DEBUG nova.compute.utils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 930.850606] env[62499]: DEBUG nova.compute.manager [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 930.850824] env[62499]: DEBUG nova.network.neutron [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 930.914482] env[62499]: DEBUG nova.policy [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ffc458db78a04358af903d6119c5eac9', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '967135c837004321824783716458da8e', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 931.209492] env[62499]: DEBUG nova.compute.manager [req-40d6addc-47ba-4409-8f3f-8b42fb607b20 req-e32f5c7b-9692-4076-a97e-d289f9490c81 service nova] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Received event network-changed-17a49625-1200-4a66-81d3-8a6e144d9234 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 931.209752] env[62499]: DEBUG nova.compute.manager [req-40d6addc-47ba-4409-8f3f-8b42fb607b20 req-e32f5c7b-9692-4076-a97e-d289f9490c81 service nova] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Refreshing instance network info cache due to event network-changed-17a49625-1200-4a66-81d3-8a6e144d9234. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 931.209925] env[62499]: DEBUG oslo_concurrency.lockutils [req-40d6addc-47ba-4409-8f3f-8b42fb607b20 req-e32f5c7b-9692-4076-a97e-d289f9490c81 service nova] Acquiring lock "refresh_cache-88466440-7392-43d3-83ec-45229516a3ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 931.225027] env[62499]: DEBUG nova.network.neutron [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Successfully created port: d984b948-6a63-4e30-b2af-b83378f4d62b {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 931.290459] env[62499]: DEBUG oslo_concurrency.lockutils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Releasing lock "refresh_cache-88466440-7392-43d3-83ec-45229516a3ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 931.290459] env[62499]: DEBUG nova.compute.manager [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 931.290459] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 931.290459] env[62499]: DEBUG oslo_concurrency.lockutils [req-40d6addc-47ba-4409-8f3f-8b42fb607b20 req-e32f5c7b-9692-4076-a97e-d289f9490c81 service nova] Acquired lock "refresh_cache-88466440-7392-43d3-83ec-45229516a3ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 931.290674] env[62499]: DEBUG nova.network.neutron [req-40d6addc-47ba-4409-8f3f-8b42fb607b20 req-e32f5c7b-9692-4076-a97e-d289f9490c81 service nova] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Refreshing network info cache for port 17a49625-1200-4a66-81d3-8a6e144d9234 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 931.291710] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f10a8d40-cbd0-40a9-838b-c6cb79ed474e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 931.301803] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e51525b3-8f85-460d-9fff-31d2052e1dbe {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 931.323084] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 88466440-7392-43d3-83ec-45229516a3ef could not be found. [ 931.323251] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 931.323445] env[62499]: INFO nova.compute.manager [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Took 0.03 seconds to destroy the instance on the hypervisor. [ 931.323696] env[62499]: DEBUG oslo.service.loopingcall [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 931.323899] env[62499]: DEBUG nova.compute.manager [-] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 931.323987] env[62499]: DEBUG nova.network.neutron [-] [instance: 88466440-7392-43d3-83ec-45229516a3ef] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 931.341135] env[62499]: DEBUG nova.network.neutron [-] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 931.351866] env[62499]: DEBUG nova.compute.manager [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 931.487268] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "ca937041-1aa6-46d8-be4e-b4983652fe15" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 931.487609] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "ca937041-1aa6-46d8-be4e-b4983652fe15" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.004s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 931.538471] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-773e71e2-6e1f-4a1b-8a46-d68901200a32 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 931.546811] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b72ef170-8e54-4320-92eb-964f98998853 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 931.578227] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e98003db-0d14-48fc-9cb4-c57b8a383db2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 931.585132] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2fe06e9f-a721-4f35-b885-f8d6075b154a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 931.598298] env[62499]: DEBUG nova.compute.provider_tree [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 931.809884] env[62499]: DEBUG nova.network.neutron [req-40d6addc-47ba-4409-8f3f-8b42fb607b20 req-e32f5c7b-9692-4076-a97e-d289f9490c81 service nova] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 931.844227] env[62499]: DEBUG nova.network.neutron [-] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 931.909849] env[62499]: DEBUG nova.network.neutron [req-40d6addc-47ba-4409-8f3f-8b42fb607b20 req-e32f5c7b-9692-4076-a97e-d289f9490c81 service nova] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 931.993201] env[62499]: DEBUG nova.compute.manager [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 932.094376] env[62499]: ERROR nova.compute.manager [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d984b948-6a63-4e30-b2af-b83378f4d62b, please check neutron logs for more information. [ 932.094376] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 932.094376] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 932.094376] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 932.094376] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 932.094376] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 932.094376] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 932.094376] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 932.094376] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 932.094376] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 932.094376] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 932.094376] env[62499]: ERROR nova.compute.manager raise self.value [ 932.094376] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 932.094376] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 932.094376] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 932.094376] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 932.095009] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 932.095009] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 932.095009] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d984b948-6a63-4e30-b2af-b83378f4d62b, please check neutron logs for more information. [ 932.095009] env[62499]: ERROR nova.compute.manager [ 932.095009] env[62499]: Traceback (most recent call last): [ 932.095009] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 932.095009] env[62499]: listener.cb(fileno) [ 932.095009] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 932.095009] env[62499]: result = function(*args, **kwargs) [ 932.095009] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 932.095009] env[62499]: return func(*args, **kwargs) [ 932.095009] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 932.095009] env[62499]: raise e [ 932.095009] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 932.095009] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 932.095009] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 932.095009] env[62499]: created_port_ids = self._update_ports_for_instance( [ 932.095009] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 932.095009] env[62499]: with excutils.save_and_reraise_exception(): [ 932.095009] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 932.095009] env[62499]: self.force_reraise() [ 932.095009] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 932.095009] env[62499]: raise self.value [ 932.095009] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 932.095009] env[62499]: updated_port = self._update_port( [ 932.095009] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 932.095009] env[62499]: _ensure_no_port_binding_failure(port) [ 932.095009] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 932.095009] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 932.095926] env[62499]: nova.exception.PortBindingFailed: Binding failed for port d984b948-6a63-4e30-b2af-b83378f4d62b, please check neutron logs for more information. [ 932.095926] env[62499]: Removing descriptor: 16 [ 932.101075] env[62499]: DEBUG nova.scheduler.client.report [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 932.348030] env[62499]: INFO nova.compute.manager [-] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Took 1.02 seconds to deallocate network for instance. [ 932.350537] env[62499]: DEBUG nova.compute.claims [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 932.350719] env[62499]: DEBUG oslo_concurrency.lockutils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 932.367030] env[62499]: DEBUG nova.compute.manager [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 932.392066] env[62499]: DEBUG nova.virt.hardware [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 932.392311] env[62499]: DEBUG nova.virt.hardware [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 932.392466] env[62499]: DEBUG nova.virt.hardware [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 932.392641] env[62499]: DEBUG nova.virt.hardware [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 932.392784] env[62499]: DEBUG nova.virt.hardware [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 932.392929] env[62499]: DEBUG nova.virt.hardware [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 932.393152] env[62499]: DEBUG nova.virt.hardware [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 932.393309] env[62499]: DEBUG nova.virt.hardware [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 932.393471] env[62499]: DEBUG nova.virt.hardware [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 932.393627] env[62499]: DEBUG nova.virt.hardware [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 932.393791] env[62499]: DEBUG nova.virt.hardware [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 932.394644] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b893f82-91c4-43d5-90ed-057dcc8edcc8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 932.402597] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e1e86ded-dcad-43c9-b4d1-99decae7597f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 932.415703] env[62499]: DEBUG oslo_concurrency.lockutils [req-40d6addc-47ba-4409-8f3f-8b42fb607b20 req-e32f5c7b-9692-4076-a97e-d289f9490c81 service nova] Releasing lock "refresh_cache-88466440-7392-43d3-83ec-45229516a3ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 932.415945] env[62499]: DEBUG nova.compute.manager [req-40d6addc-47ba-4409-8f3f-8b42fb607b20 req-e32f5c7b-9692-4076-a97e-d289f9490c81 service nova] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Received event network-vif-deleted-17a49625-1200-4a66-81d3-8a6e144d9234 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 932.416834] env[62499]: ERROR nova.compute.manager [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d984b948-6a63-4e30-b2af-b83378f4d62b, please check neutron logs for more information. [ 932.416834] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Traceback (most recent call last): [ 932.416834] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 932.416834] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] yield resources [ 932.416834] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 932.416834] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] self.driver.spawn(context, instance, image_meta, [ 932.416834] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 932.416834] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] self._vmops.spawn(context, instance, image_meta, injected_files, [ 932.416834] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 932.416834] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] vm_ref = self.build_virtual_machine(instance, [ 932.416834] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 932.417186] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] vif_infos = vmwarevif.get_vif_info(self._session, [ 932.417186] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 932.417186] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] for vif in network_info: [ 932.417186] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 932.417186] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] return self._sync_wrapper(fn, *args, **kwargs) [ 932.417186] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 932.417186] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] self.wait() [ 932.417186] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 932.417186] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] self[:] = self._gt.wait() [ 932.417186] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 932.417186] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] return self._exit_event.wait() [ 932.417186] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 932.417186] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] current.throw(*self._exc) [ 932.417574] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 932.417574] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] result = function(*args, **kwargs) [ 932.417574] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 932.417574] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] return func(*args, **kwargs) [ 932.417574] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 932.417574] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] raise e [ 932.417574] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 932.417574] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] nwinfo = self.network_api.allocate_for_instance( [ 932.417574] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 932.417574] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] created_port_ids = self._update_ports_for_instance( [ 932.417574] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 932.417574] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] with excutils.save_and_reraise_exception(): [ 932.417574] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 932.417954] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] self.force_reraise() [ 932.417954] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 932.417954] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] raise self.value [ 932.417954] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 932.417954] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] updated_port = self._update_port( [ 932.417954] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 932.417954] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] _ensure_no_port_binding_failure(port) [ 932.417954] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 932.417954] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] raise exception.PortBindingFailed(port_id=port['id']) [ 932.417954] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] nova.exception.PortBindingFailed: Binding failed for port d984b948-6a63-4e30-b2af-b83378f4d62b, please check neutron logs for more information. [ 932.417954] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] [ 932.417954] env[62499]: INFO nova.compute.manager [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Terminating instance [ 932.419066] env[62499]: DEBUG oslo_concurrency.lockutils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Acquiring lock "refresh_cache-95a5cb2c-24da-43c0-b9d1-7f0b51ccda87" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 932.419225] env[62499]: DEBUG oslo_concurrency.lockutils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Acquired lock "refresh_cache-95a5cb2c-24da-43c0-b9d1-7f0b51ccda87" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 932.419498] env[62499]: DEBUG nova.network.neutron [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 932.514361] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 932.605422] env[62499]: DEBUG oslo_concurrency.lockutils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.273s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 932.605919] env[62499]: DEBUG nova.compute.manager [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 932.608828] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.140s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 932.610188] env[62499]: INFO nova.compute.claims [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 932.935917] env[62499]: DEBUG nova.network.neutron [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 933.024537] env[62499]: DEBUG nova.network.neutron [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 933.114228] env[62499]: DEBUG nova.compute.utils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 933.117719] env[62499]: DEBUG nova.compute.manager [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 933.117719] env[62499]: DEBUG nova.network.neutron [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 933.158191] env[62499]: DEBUG nova.policy [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c92b9d37f796479ca3b6d9f0ed1789c8', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e444950a637b4fdab2edb1d62aac4be5', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 933.238788] env[62499]: DEBUG nova.compute.manager [req-26d04441-6660-4df1-a031-a6a45a4295d6 req-87c7125f-9386-436c-a523-a9bf57426fd6 service nova] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Received event network-changed-d984b948-6a63-4e30-b2af-b83378f4d62b {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 933.239851] env[62499]: DEBUG nova.compute.manager [req-26d04441-6660-4df1-a031-a6a45a4295d6 req-87c7125f-9386-436c-a523-a9bf57426fd6 service nova] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Refreshing instance network info cache due to event network-changed-d984b948-6a63-4e30-b2af-b83378f4d62b. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 933.240197] env[62499]: DEBUG oslo_concurrency.lockutils [req-26d04441-6660-4df1-a031-a6a45a4295d6 req-87c7125f-9386-436c-a523-a9bf57426fd6 service nova] Acquiring lock "refresh_cache-95a5cb2c-24da-43c0-b9d1-7f0b51ccda87" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 933.417072] env[62499]: DEBUG nova.network.neutron [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Successfully created port: e5633ef6-c4da-4c0c-bf53-6f3e9160d31a {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 933.528147] env[62499]: DEBUG oslo_concurrency.lockutils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Releasing lock "refresh_cache-95a5cb2c-24da-43c0-b9d1-7f0b51ccda87" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 933.528602] env[62499]: DEBUG nova.compute.manager [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 933.528796] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 933.529129] env[62499]: DEBUG oslo_concurrency.lockutils [req-26d04441-6660-4df1-a031-a6a45a4295d6 req-87c7125f-9386-436c-a523-a9bf57426fd6 service nova] Acquired lock "refresh_cache-95a5cb2c-24da-43c0-b9d1-7f0b51ccda87" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 933.529306] env[62499]: DEBUG nova.network.neutron [req-26d04441-6660-4df1-a031-a6a45a4295d6 req-87c7125f-9386-436c-a523-a9bf57426fd6 service nova] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Refreshing network info cache for port d984b948-6a63-4e30-b2af-b83378f4d62b {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 933.530941] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d266c0a7-9930-44db-a21b-f881d1968189 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.540081] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ddbd0df-dc99-4268-9ba9-e4057a4a99c0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.563766] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87 could not be found. [ 933.563766] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 933.563766] env[62499]: INFO nova.compute.manager [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Took 0.03 seconds to destroy the instance on the hypervisor. [ 933.563766] env[62499]: DEBUG oslo.service.loopingcall [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 933.563766] env[62499]: DEBUG nova.compute.manager [-] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 933.563766] env[62499]: DEBUG nova.network.neutron [-] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 933.577390] env[62499]: DEBUG nova.network.neutron [-] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 933.618829] env[62499]: DEBUG nova.compute.manager [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 933.795602] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78a1c741-3e99-4da2-bfec-1c91325ca779 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.803309] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b70a548f-b007-4a2c-a415-ed87ece86140 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.833097] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-918aebee-3f50-4fce-9af9-d1b4bc8dece3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.840041] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55e96e06-c453-41d3-b87e-8e63a16bb6b1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.853652] env[62499]: DEBUG nova.compute.provider_tree [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 934.050712] env[62499]: DEBUG nova.network.neutron [req-26d04441-6660-4df1-a031-a6a45a4295d6 req-87c7125f-9386-436c-a523-a9bf57426fd6 service nova] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 934.080310] env[62499]: DEBUG nova.network.neutron [-] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 934.134156] env[62499]: DEBUG nova.network.neutron [req-26d04441-6660-4df1-a031-a6a45a4295d6 req-87c7125f-9386-436c-a523-a9bf57426fd6 service nova] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 934.267713] env[62499]: ERROR nova.compute.manager [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port e5633ef6-c4da-4c0c-bf53-6f3e9160d31a, please check neutron logs for more information. [ 934.267713] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 934.267713] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 934.267713] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 934.267713] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 934.267713] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 934.267713] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 934.267713] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 934.267713] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 934.267713] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 934.267713] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 934.267713] env[62499]: ERROR nova.compute.manager raise self.value [ 934.267713] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 934.267713] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 934.267713] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 934.267713] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 934.268244] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 934.268244] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 934.268244] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port e5633ef6-c4da-4c0c-bf53-6f3e9160d31a, please check neutron logs for more information. [ 934.268244] env[62499]: ERROR nova.compute.manager [ 934.268244] env[62499]: Traceback (most recent call last): [ 934.268244] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 934.268244] env[62499]: listener.cb(fileno) [ 934.268244] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 934.268244] env[62499]: result = function(*args, **kwargs) [ 934.268244] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 934.268244] env[62499]: return func(*args, **kwargs) [ 934.268244] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 934.268244] env[62499]: raise e [ 934.268244] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 934.268244] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 934.268244] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 934.268244] env[62499]: created_port_ids = self._update_ports_for_instance( [ 934.268244] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 934.268244] env[62499]: with excutils.save_and_reraise_exception(): [ 934.268244] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 934.268244] env[62499]: self.force_reraise() [ 934.268244] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 934.268244] env[62499]: raise self.value [ 934.268244] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 934.268244] env[62499]: updated_port = self._update_port( [ 934.268244] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 934.268244] env[62499]: _ensure_no_port_binding_failure(port) [ 934.268244] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 934.268244] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 934.269094] env[62499]: nova.exception.PortBindingFailed: Binding failed for port e5633ef6-c4da-4c0c-bf53-6f3e9160d31a, please check neutron logs for more information. [ 934.269094] env[62499]: Removing descriptor: 16 [ 934.355666] env[62499]: DEBUG nova.scheduler.client.report [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 934.582774] env[62499]: INFO nova.compute.manager [-] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Took 1.02 seconds to deallocate network for instance. [ 934.585052] env[62499]: DEBUG nova.compute.claims [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 934.585241] env[62499]: DEBUG oslo_concurrency.lockutils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 934.630666] env[62499]: DEBUG nova.compute.manager [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 934.636147] env[62499]: DEBUG oslo_concurrency.lockutils [req-26d04441-6660-4df1-a031-a6a45a4295d6 req-87c7125f-9386-436c-a523-a9bf57426fd6 service nova] Releasing lock "refresh_cache-95a5cb2c-24da-43c0-b9d1-7f0b51ccda87" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 934.636374] env[62499]: DEBUG nova.compute.manager [req-26d04441-6660-4df1-a031-a6a45a4295d6 req-87c7125f-9386-436c-a523-a9bf57426fd6 service nova] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Received event network-vif-deleted-d984b948-6a63-4e30-b2af-b83378f4d62b {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 934.655520] env[62499]: DEBUG nova.virt.hardware [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 934.655750] env[62499]: DEBUG nova.virt.hardware [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 934.655904] env[62499]: DEBUG nova.virt.hardware [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 934.656090] env[62499]: DEBUG nova.virt.hardware [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 934.656240] env[62499]: DEBUG nova.virt.hardware [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 934.656386] env[62499]: DEBUG nova.virt.hardware [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 934.656581] env[62499]: DEBUG nova.virt.hardware [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 934.656748] env[62499]: DEBUG nova.virt.hardware [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 934.656918] env[62499]: DEBUG nova.virt.hardware [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 934.657090] env[62499]: DEBUG nova.virt.hardware [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 934.657264] env[62499]: DEBUG nova.virt.hardware [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 934.658154] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6fdf7c67-04ee-410d-8a11-891a0a2cac23 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 934.666047] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a52f452-6794-4914-b94e-6664485a415f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 934.680692] env[62499]: ERROR nova.compute.manager [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port e5633ef6-c4da-4c0c-bf53-6f3e9160d31a, please check neutron logs for more information. [ 934.680692] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Traceback (most recent call last): [ 934.680692] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 934.680692] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] yield resources [ 934.680692] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 934.680692] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] self.driver.spawn(context, instance, image_meta, [ 934.680692] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 934.680692] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] self._vmops.spawn(context, instance, image_meta, injected_files, [ 934.680692] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 934.680692] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] vm_ref = self.build_virtual_machine(instance, [ 934.680692] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 934.681096] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] vif_infos = vmwarevif.get_vif_info(self._session, [ 934.681096] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 934.681096] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] for vif in network_info: [ 934.681096] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 934.681096] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] return self._sync_wrapper(fn, *args, **kwargs) [ 934.681096] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 934.681096] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] self.wait() [ 934.681096] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 934.681096] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] self[:] = self._gt.wait() [ 934.681096] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 934.681096] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] return self._exit_event.wait() [ 934.681096] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 934.681096] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] current.throw(*self._exc) [ 934.681481] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 934.681481] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] result = function(*args, **kwargs) [ 934.681481] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 934.681481] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] return func(*args, **kwargs) [ 934.681481] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 934.681481] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] raise e [ 934.681481] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 934.681481] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] nwinfo = self.network_api.allocate_for_instance( [ 934.681481] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 934.681481] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] created_port_ids = self._update_ports_for_instance( [ 934.681481] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 934.681481] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] with excutils.save_and_reraise_exception(): [ 934.681481] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 934.681864] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] self.force_reraise() [ 934.681864] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 934.681864] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] raise self.value [ 934.681864] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 934.681864] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] updated_port = self._update_port( [ 934.681864] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 934.681864] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] _ensure_no_port_binding_failure(port) [ 934.681864] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 934.681864] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] raise exception.PortBindingFailed(port_id=port['id']) [ 934.681864] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] nova.exception.PortBindingFailed: Binding failed for port e5633ef6-c4da-4c0c-bf53-6f3e9160d31a, please check neutron logs for more information. [ 934.681864] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] [ 934.681864] env[62499]: INFO nova.compute.manager [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Terminating instance [ 934.682910] env[62499]: DEBUG oslo_concurrency.lockutils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "refresh_cache-9fd7db7e-466e-4213-9e1d-210eae714884" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 934.683088] env[62499]: DEBUG oslo_concurrency.lockutils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquired lock "refresh_cache-9fd7db7e-466e-4213-9e1d-210eae714884" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 934.683257] env[62499]: DEBUG nova.network.neutron [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 934.860855] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.252s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 934.861579] env[62499]: DEBUG nova.compute.manager [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 934.864633] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 13.904s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 934.864813] env[62499]: DEBUG nova.objects.instance [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] [instance: b534fcdb-b0db-4e07-85fc-696dc58168aa] Trying to apply a migration context that does not seem to be set for this instance {{(pid=62499) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 935.199582] env[62499]: DEBUG nova.network.neutron [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 935.263986] env[62499]: DEBUG nova.compute.manager [req-c46b87ec-79ef-43ee-9063-c97d16443546 req-24ce5ff2-70c6-4e37-aa7a-693cd0e2607d service nova] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Received event network-changed-e5633ef6-c4da-4c0c-bf53-6f3e9160d31a {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 935.264194] env[62499]: DEBUG nova.compute.manager [req-c46b87ec-79ef-43ee-9063-c97d16443546 req-24ce5ff2-70c6-4e37-aa7a-693cd0e2607d service nova] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Refreshing instance network info cache due to event network-changed-e5633ef6-c4da-4c0c-bf53-6f3e9160d31a. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 935.264386] env[62499]: DEBUG oslo_concurrency.lockutils [req-c46b87ec-79ef-43ee-9063-c97d16443546 req-24ce5ff2-70c6-4e37-aa7a-693cd0e2607d service nova] Acquiring lock "refresh_cache-9fd7db7e-466e-4213-9e1d-210eae714884" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 935.287168] env[62499]: DEBUG nova.network.neutron [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 935.370324] env[62499]: DEBUG nova.compute.utils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 935.374169] env[62499]: DEBUG nova.compute.manager [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 935.374360] env[62499]: DEBUG nova.network.neutron [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 935.412933] env[62499]: DEBUG nova.policy [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7da3078e8bb5444c96e007cfc2d57952', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c9f95d7e954f4413b01aad519c498967', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 935.790433] env[62499]: DEBUG oslo_concurrency.lockutils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Releasing lock "refresh_cache-9fd7db7e-466e-4213-9e1d-210eae714884" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 935.790433] env[62499]: DEBUG nova.compute.manager [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 935.790724] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 935.791641] env[62499]: DEBUG oslo_concurrency.lockutils [req-c46b87ec-79ef-43ee-9063-c97d16443546 req-24ce5ff2-70c6-4e37-aa7a-693cd0e2607d service nova] Acquired lock "refresh_cache-9fd7db7e-466e-4213-9e1d-210eae714884" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 935.791641] env[62499]: DEBUG nova.network.neutron [req-c46b87ec-79ef-43ee-9063-c97d16443546 req-24ce5ff2-70c6-4e37-aa7a-693cd0e2607d service nova] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Refreshing network info cache for port e5633ef6-c4da-4c0c-bf53-6f3e9160d31a {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 935.791980] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d154ceed-1ade-4a12-8c75-47188d1082ba {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 935.802886] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83284084-73ec-4d52-b1be-4ed58131fa74 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 935.816975] env[62499]: DEBUG nova.network.neutron [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Successfully created port: 4464326a-eda6-4130-9df6-a9d552a560c1 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 935.825060] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9fd7db7e-466e-4213-9e1d-210eae714884 could not be found. [ 935.825270] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 935.825442] env[62499]: INFO nova.compute.manager [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Took 0.03 seconds to destroy the instance on the hypervisor. [ 935.825675] env[62499]: DEBUG oslo.service.loopingcall [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 935.825892] env[62499]: DEBUG nova.compute.manager [-] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 935.825983] env[62499]: DEBUG nova.network.neutron [-] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 935.840628] env[62499]: DEBUG nova.network.neutron [-] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 935.874864] env[62499]: DEBUG nova.compute.manager [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 935.878803] env[62499]: DEBUG oslo_concurrency.lockutils [None req-dd59643a-3941-445d-89d8-7097abf1cd62 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.014s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 935.879149] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.514s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 935.880716] env[62499]: INFO nova.compute.claims [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 936.313552] env[62499]: DEBUG nova.network.neutron [req-c46b87ec-79ef-43ee-9063-c97d16443546 req-24ce5ff2-70c6-4e37-aa7a-693cd0e2607d service nova] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 936.343899] env[62499]: DEBUG nova.network.neutron [-] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 936.417133] env[62499]: DEBUG nova.network.neutron [req-c46b87ec-79ef-43ee-9063-c97d16443546 req-24ce5ff2-70c6-4e37-aa7a-693cd0e2607d service nova] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 936.681142] env[62499]: ERROR nova.compute.manager [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 4464326a-eda6-4130-9df6-a9d552a560c1, please check neutron logs for more information. [ 936.681142] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 936.681142] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 936.681142] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 936.681142] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 936.681142] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 936.681142] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 936.681142] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 936.681142] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 936.681142] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 936.681142] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 936.681142] env[62499]: ERROR nova.compute.manager raise self.value [ 936.681142] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 936.681142] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 936.681142] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 936.681142] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 936.681708] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 936.681708] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 936.681708] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 4464326a-eda6-4130-9df6-a9d552a560c1, please check neutron logs for more information. [ 936.681708] env[62499]: ERROR nova.compute.manager [ 936.681708] env[62499]: Traceback (most recent call last): [ 936.681708] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 936.681708] env[62499]: listener.cb(fileno) [ 936.681708] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 936.681708] env[62499]: result = function(*args, **kwargs) [ 936.681708] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 936.681708] env[62499]: return func(*args, **kwargs) [ 936.681708] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 936.681708] env[62499]: raise e [ 936.681708] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 936.681708] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 936.681708] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 936.681708] env[62499]: created_port_ids = self._update_ports_for_instance( [ 936.681708] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 936.681708] env[62499]: with excutils.save_and_reraise_exception(): [ 936.681708] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 936.681708] env[62499]: self.force_reraise() [ 936.681708] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 936.681708] env[62499]: raise self.value [ 936.681708] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 936.681708] env[62499]: updated_port = self._update_port( [ 936.681708] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 936.681708] env[62499]: _ensure_no_port_binding_failure(port) [ 936.681708] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 936.681708] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 936.682589] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 4464326a-eda6-4130-9df6-a9d552a560c1, please check neutron logs for more information. [ 936.682589] env[62499]: Removing descriptor: 16 [ 936.846725] env[62499]: INFO nova.compute.manager [-] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Took 1.02 seconds to deallocate network for instance. [ 936.849708] env[62499]: DEBUG nova.compute.claims [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 936.849708] env[62499]: DEBUG oslo_concurrency.lockutils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 936.883133] env[62499]: DEBUG nova.compute.manager [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 936.911887] env[62499]: DEBUG nova.virt.hardware [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 936.912153] env[62499]: DEBUG nova.virt.hardware [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 936.912329] env[62499]: DEBUG nova.virt.hardware [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 936.912530] env[62499]: DEBUG nova.virt.hardware [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 936.912679] env[62499]: DEBUG nova.virt.hardware [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 936.912827] env[62499]: DEBUG nova.virt.hardware [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 936.913057] env[62499]: DEBUG nova.virt.hardware [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 936.913225] env[62499]: DEBUG nova.virt.hardware [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 936.913390] env[62499]: DEBUG nova.virt.hardware [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 936.913552] env[62499]: DEBUG nova.virt.hardware [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 936.913720] env[62499]: DEBUG nova.virt.hardware [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 936.914854] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d9bbc322-8ca0-4f89-a57e-6a4097bcaf50 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 936.917495] env[62499]: DEBUG oslo_concurrency.lockutils [req-c46b87ec-79ef-43ee-9063-c97d16443546 req-24ce5ff2-70c6-4e37-aa7a-693cd0e2607d service nova] Releasing lock "refresh_cache-9fd7db7e-466e-4213-9e1d-210eae714884" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 936.917719] env[62499]: DEBUG nova.compute.manager [req-c46b87ec-79ef-43ee-9063-c97d16443546 req-24ce5ff2-70c6-4e37-aa7a-693cd0e2607d service nova] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Received event network-vif-deleted-e5633ef6-c4da-4c0c-bf53-6f3e9160d31a {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 936.923092] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f40ab9c-6fe2-474b-b207-92c5ba8cfe40 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 936.940923] env[62499]: ERROR nova.compute.manager [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 4464326a-eda6-4130-9df6-a9d552a560c1, please check neutron logs for more information. [ 936.940923] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Traceback (most recent call last): [ 936.940923] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 936.940923] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] yield resources [ 936.940923] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 936.940923] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] self.driver.spawn(context, instance, image_meta, [ 936.940923] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 936.940923] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] self._vmops.spawn(context, instance, image_meta, injected_files, [ 936.940923] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 936.940923] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] vm_ref = self.build_virtual_machine(instance, [ 936.940923] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 936.941393] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] vif_infos = vmwarevif.get_vif_info(self._session, [ 936.941393] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 936.941393] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] for vif in network_info: [ 936.941393] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 936.941393] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] return self._sync_wrapper(fn, *args, **kwargs) [ 936.941393] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 936.941393] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] self.wait() [ 936.941393] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 936.941393] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] self[:] = self._gt.wait() [ 936.941393] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 936.941393] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] return self._exit_event.wait() [ 936.941393] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 936.941393] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] current.throw(*self._exc) [ 936.941769] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 936.941769] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] result = function(*args, **kwargs) [ 936.941769] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 936.941769] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] return func(*args, **kwargs) [ 936.941769] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 936.941769] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] raise e [ 936.941769] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 936.941769] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] nwinfo = self.network_api.allocate_for_instance( [ 936.941769] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 936.941769] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] created_port_ids = self._update_ports_for_instance( [ 936.941769] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 936.941769] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] with excutils.save_and_reraise_exception(): [ 936.941769] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 936.942160] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] self.force_reraise() [ 936.942160] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 936.942160] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] raise self.value [ 936.942160] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 936.942160] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] updated_port = self._update_port( [ 936.942160] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 936.942160] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] _ensure_no_port_binding_failure(port) [ 936.942160] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 936.942160] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] raise exception.PortBindingFailed(port_id=port['id']) [ 936.942160] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] nova.exception.PortBindingFailed: Binding failed for port 4464326a-eda6-4130-9df6-a9d552a560c1, please check neutron logs for more information. [ 936.942160] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] [ 936.942160] env[62499]: INFO nova.compute.manager [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Terminating instance [ 936.945267] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "refresh_cache-ea4b6c62-2880-4a59-8017-b5d1518c1060" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 936.945434] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquired lock "refresh_cache-ea4b6c62-2880-4a59-8017-b5d1518c1060" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 936.945585] env[62499]: DEBUG nova.network.neutron [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 937.036368] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba52ea16-d591-4986-a4c5-7c254843189b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.043867] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58d27661-ab8a-44cb-bdce-8fcaac8d6ac7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.071934] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f6a35f55-9d40-4c7a-acfe-b2531266ebc9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.078630] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-849b0a80-b675-4e31-b301-147768b4ec0c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.091158] env[62499]: DEBUG nova.compute.provider_tree [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 937.286879] env[62499]: DEBUG nova.compute.manager [req-66f27107-c1d2-4690-9016-018db2180b64 req-b0dd36ff-1bd2-4322-9b63-b069a8d0db9d service nova] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Received event network-changed-4464326a-eda6-4130-9df6-a9d552a560c1 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 937.287088] env[62499]: DEBUG nova.compute.manager [req-66f27107-c1d2-4690-9016-018db2180b64 req-b0dd36ff-1bd2-4322-9b63-b069a8d0db9d service nova] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Refreshing instance network info cache due to event network-changed-4464326a-eda6-4130-9df6-a9d552a560c1. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 937.287277] env[62499]: DEBUG oslo_concurrency.lockutils [req-66f27107-c1d2-4690-9016-018db2180b64 req-b0dd36ff-1bd2-4322-9b63-b069a8d0db9d service nova] Acquiring lock "refresh_cache-ea4b6c62-2880-4a59-8017-b5d1518c1060" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 937.463469] env[62499]: DEBUG nova.network.neutron [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 937.561404] env[62499]: DEBUG nova.network.neutron [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 937.593732] env[62499]: DEBUG nova.scheduler.client.report [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 938.064586] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Releasing lock "refresh_cache-ea4b6c62-2880-4a59-8017-b5d1518c1060" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 938.065049] env[62499]: DEBUG nova.compute.manager [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 938.065244] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 938.065551] env[62499]: DEBUG oslo_concurrency.lockutils [req-66f27107-c1d2-4690-9016-018db2180b64 req-b0dd36ff-1bd2-4322-9b63-b069a8d0db9d service nova] Acquired lock "refresh_cache-ea4b6c62-2880-4a59-8017-b5d1518c1060" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 938.065719] env[62499]: DEBUG nova.network.neutron [req-66f27107-c1d2-4690-9016-018db2180b64 req-b0dd36ff-1bd2-4322-9b63-b069a8d0db9d service nova] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Refreshing network info cache for port 4464326a-eda6-4130-9df6-a9d552a560c1 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 938.066795] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-14f10709-c685-44d5-a345-54febf394474 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 938.076022] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-87d4c59a-2c08-4f60-ab40-62dd4d4d5073 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 938.098928] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.220s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 938.099473] env[62499]: DEBUG nova.compute.manager [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 938.103058] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ea4b6c62-2880-4a59-8017-b5d1518c1060 could not be found. [ 938.103058] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 938.103058] env[62499]: INFO nova.compute.manager [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Took 0.04 seconds to destroy the instance on the hypervisor. [ 938.103058] env[62499]: DEBUG oslo.service.loopingcall [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 938.103339] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.990s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 938.105915] env[62499]: DEBUG nova.compute.manager [-] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 938.106027] env[62499]: DEBUG nova.network.neutron [-] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 938.121681] env[62499]: DEBUG nova.network.neutron [-] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 938.593642] env[62499]: DEBUG nova.network.neutron [req-66f27107-c1d2-4690-9016-018db2180b64 req-b0dd36ff-1bd2-4322-9b63-b069a8d0db9d service nova] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 938.605007] env[62499]: DEBUG nova.compute.utils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 938.606692] env[62499]: DEBUG nova.compute.manager [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 938.607351] env[62499]: DEBUG nova.network.neutron [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 938.623854] env[62499]: DEBUG nova.network.neutron [-] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 938.651731] env[62499]: DEBUG nova.policy [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '88c76f25140f4de28bd719de1d5ff616', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'cb9ca6d8c8ae4352bc3c7b1daa7ba288', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 938.667152] env[62499]: DEBUG nova.network.neutron [req-66f27107-c1d2-4690-9016-018db2180b64 req-b0dd36ff-1bd2-4322-9b63-b069a8d0db9d service nova] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 938.746817] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e8f0e8a-3587-4a46-bfe1-78bc1a51b888 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 938.754387] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dc7ecdef-484f-4e3d-a69c-a4d9c68e61ca {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 938.785686] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45ab78de-1886-40f7-ac3d-300b9fbc672b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 938.794558] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a4b9e05-c166-4f2d-b444-8582fa8bef5e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 938.807094] env[62499]: DEBUG nova.compute.provider_tree [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 938.951928] env[62499]: DEBUG nova.network.neutron [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Successfully created port: 461e9d84-836a-4f8b-8ea2-f4ea9f208eef {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 939.109606] env[62499]: DEBUG nova.compute.manager [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 939.128142] env[62499]: INFO nova.compute.manager [-] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Took 1.02 seconds to deallocate network for instance. [ 939.130503] env[62499]: DEBUG nova.compute.claims [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 939.130691] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 939.170198] env[62499]: DEBUG oslo_concurrency.lockutils [req-66f27107-c1d2-4690-9016-018db2180b64 req-b0dd36ff-1bd2-4322-9b63-b069a8d0db9d service nova] Releasing lock "refresh_cache-ea4b6c62-2880-4a59-8017-b5d1518c1060" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 939.170476] env[62499]: DEBUG nova.compute.manager [req-66f27107-c1d2-4690-9016-018db2180b64 req-b0dd36ff-1bd2-4322-9b63-b069a8d0db9d service nova] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Received event network-vif-deleted-4464326a-eda6-4130-9df6-a9d552a560c1 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 939.311627] env[62499]: DEBUG nova.scheduler.client.report [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 939.651973] env[62499]: DEBUG nova.compute.manager [req-c94b552e-fadd-48be-88ab-d7fb4e8fcb84 req-4f488584-12ac-4276-8205-9ce9a0e9d764 service nova] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Received event network-changed-461e9d84-836a-4f8b-8ea2-f4ea9f208eef {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 939.652971] env[62499]: DEBUG nova.compute.manager [req-c94b552e-fadd-48be-88ab-d7fb4e8fcb84 req-4f488584-12ac-4276-8205-9ce9a0e9d764 service nova] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Refreshing instance network info cache due to event network-changed-461e9d84-836a-4f8b-8ea2-f4ea9f208eef. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 939.652971] env[62499]: DEBUG oslo_concurrency.lockutils [req-c94b552e-fadd-48be-88ab-d7fb4e8fcb84 req-4f488584-12ac-4276-8205-9ce9a0e9d764 service nova] Acquiring lock "refresh_cache-99920efd-13e6-4a18-9532-4b53bea3c736" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 939.652971] env[62499]: DEBUG oslo_concurrency.lockutils [req-c94b552e-fadd-48be-88ab-d7fb4e8fcb84 req-4f488584-12ac-4276-8205-9ce9a0e9d764 service nova] Acquired lock "refresh_cache-99920efd-13e6-4a18-9532-4b53bea3c736" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 939.652971] env[62499]: DEBUG nova.network.neutron [req-c94b552e-fadd-48be-88ab-d7fb4e8fcb84 req-4f488584-12ac-4276-8205-9ce9a0e9d764 service nova] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Refreshing network info cache for port 461e9d84-836a-4f8b-8ea2-f4ea9f208eef {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 939.816969] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.713s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 939.817644] env[62499]: ERROR nova.compute.manager [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b2333580-9163-4b2c-bd7f-1a2225de00b4, please check neutron logs for more information. [ 939.817644] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Traceback (most recent call last): [ 939.817644] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 939.817644] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] self.driver.spawn(context, instance, image_meta, [ 939.817644] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 939.817644] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 939.817644] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 939.817644] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] vm_ref = self.build_virtual_machine(instance, [ 939.817644] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 939.817644] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] vif_infos = vmwarevif.get_vif_info(self._session, [ 939.817644] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 939.817965] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] for vif in network_info: [ 939.817965] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 939.817965] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] return self._sync_wrapper(fn, *args, **kwargs) [ 939.817965] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 939.817965] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] self.wait() [ 939.817965] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 939.817965] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] self[:] = self._gt.wait() [ 939.817965] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 939.817965] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] return self._exit_event.wait() [ 939.817965] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 939.817965] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] result = hub.switch() [ 939.817965] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 939.817965] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] return self.greenlet.switch() [ 939.818344] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 939.818344] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] result = function(*args, **kwargs) [ 939.818344] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 939.818344] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] return func(*args, **kwargs) [ 939.818344] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 939.818344] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] raise e [ 939.818344] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 939.818344] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] nwinfo = self.network_api.allocate_for_instance( [ 939.818344] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 939.818344] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] created_port_ids = self._update_ports_for_instance( [ 939.818344] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 939.818344] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] with excutils.save_and_reraise_exception(): [ 939.818344] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 939.818725] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] self.force_reraise() [ 939.818725] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 939.818725] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] raise self.value [ 939.818725] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 939.818725] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] updated_port = self._update_port( [ 939.818725] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 939.818725] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] _ensure_no_port_binding_failure(port) [ 939.818725] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 939.818725] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] raise exception.PortBindingFailed(port_id=port['id']) [ 939.818725] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] nova.exception.PortBindingFailed: Binding failed for port b2333580-9163-4b2c-bd7f-1a2225de00b4, please check neutron logs for more information. [ 939.818725] env[62499]: ERROR nova.compute.manager [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] [ 939.819059] env[62499]: DEBUG nova.compute.utils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Binding failed for port b2333580-9163-4b2c-bd7f-1a2225de00b4, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 939.819693] env[62499]: DEBUG oslo_concurrency.lockutils [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 14.450s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 939.819930] env[62499]: DEBUG nova.objects.instance [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Lazy-loading 'resources' on Instance uuid b534fcdb-b0db-4e07-85fc-696dc58168aa {{(pid=62499) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 939.821201] env[62499]: DEBUG nova.compute.manager [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Build of instance f1b5512b-702f-45b1-9844-13a1218c31b3 was re-scheduled: Binding failed for port b2333580-9163-4b2c-bd7f-1a2225de00b4, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 939.821680] env[62499]: DEBUG nova.compute.manager [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 939.821903] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Acquiring lock "refresh_cache-f1b5512b-702f-45b1-9844-13a1218c31b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 939.822066] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Acquired lock "refresh_cache-f1b5512b-702f-45b1-9844-13a1218c31b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 939.822231] env[62499]: DEBUG nova.network.neutron [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 939.828193] env[62499]: ERROR nova.compute.manager [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 461e9d84-836a-4f8b-8ea2-f4ea9f208eef, please check neutron logs for more information. [ 939.828193] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 939.828193] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 939.828193] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 939.828193] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 939.828193] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 939.828193] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 939.828193] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 939.828193] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 939.828193] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 939.828193] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 939.828193] env[62499]: ERROR nova.compute.manager raise self.value [ 939.828193] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 939.828193] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 939.828193] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 939.828193] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 939.828664] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 939.828664] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 939.828664] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 461e9d84-836a-4f8b-8ea2-f4ea9f208eef, please check neutron logs for more information. [ 939.828664] env[62499]: ERROR nova.compute.manager [ 939.828664] env[62499]: Traceback (most recent call last): [ 939.828664] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 939.828664] env[62499]: listener.cb(fileno) [ 939.828664] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 939.828664] env[62499]: result = function(*args, **kwargs) [ 939.828664] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 939.828664] env[62499]: return func(*args, **kwargs) [ 939.828664] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 939.828664] env[62499]: raise e [ 939.828664] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 939.828664] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 939.828664] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 939.828664] env[62499]: created_port_ids = self._update_ports_for_instance( [ 939.828664] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 939.828664] env[62499]: with excutils.save_and_reraise_exception(): [ 939.828664] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 939.828664] env[62499]: self.force_reraise() [ 939.828664] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 939.828664] env[62499]: raise self.value [ 939.828664] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 939.828664] env[62499]: updated_port = self._update_port( [ 939.828664] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 939.828664] env[62499]: _ensure_no_port_binding_failure(port) [ 939.828664] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 939.828664] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 939.829515] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 461e9d84-836a-4f8b-8ea2-f4ea9f208eef, please check neutron logs for more information. [ 939.829515] env[62499]: Removing descriptor: 17 [ 940.120570] env[62499]: DEBUG nova.compute.manager [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 940.145871] env[62499]: DEBUG nova.virt.hardware [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 940.146030] env[62499]: DEBUG nova.virt.hardware [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 940.146199] env[62499]: DEBUG nova.virt.hardware [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 940.146382] env[62499]: DEBUG nova.virt.hardware [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 940.146533] env[62499]: DEBUG nova.virt.hardware [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 940.146668] env[62499]: DEBUG nova.virt.hardware [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 940.146874] env[62499]: DEBUG nova.virt.hardware [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 940.147044] env[62499]: DEBUG nova.virt.hardware [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 940.147214] env[62499]: DEBUG nova.virt.hardware [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 940.147372] env[62499]: DEBUG nova.virt.hardware [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 940.147536] env[62499]: DEBUG nova.virt.hardware [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 940.148416] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e6bc908-1a19-454c-bc8b-1d303b497dc9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 940.158826] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f950e615-29bd-4289-8f1a-f6825ab4c359 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 940.173911] env[62499]: ERROR nova.compute.manager [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 461e9d84-836a-4f8b-8ea2-f4ea9f208eef, please check neutron logs for more information. [ 940.173911] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Traceback (most recent call last): [ 940.173911] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 940.173911] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] yield resources [ 940.173911] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 940.173911] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] self.driver.spawn(context, instance, image_meta, [ 940.173911] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 940.173911] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] self._vmops.spawn(context, instance, image_meta, injected_files, [ 940.173911] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 940.173911] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] vm_ref = self.build_virtual_machine(instance, [ 940.173911] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 940.174319] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] vif_infos = vmwarevif.get_vif_info(self._session, [ 940.174319] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 940.174319] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] for vif in network_info: [ 940.174319] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 940.174319] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] return self._sync_wrapper(fn, *args, **kwargs) [ 940.174319] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 940.174319] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] self.wait() [ 940.174319] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 940.174319] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] self[:] = self._gt.wait() [ 940.174319] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 940.174319] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] return self._exit_event.wait() [ 940.174319] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 940.174319] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] current.throw(*self._exc) [ 940.174684] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 940.174684] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] result = function(*args, **kwargs) [ 940.174684] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 940.174684] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] return func(*args, **kwargs) [ 940.174684] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 940.174684] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] raise e [ 940.174684] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 940.174684] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] nwinfo = self.network_api.allocate_for_instance( [ 940.174684] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 940.174684] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] created_port_ids = self._update_ports_for_instance( [ 940.174684] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 940.174684] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] with excutils.save_and_reraise_exception(): [ 940.174684] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 940.175147] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] self.force_reraise() [ 940.175147] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 940.175147] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] raise self.value [ 940.175147] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 940.175147] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] updated_port = self._update_port( [ 940.175147] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 940.175147] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] _ensure_no_port_binding_failure(port) [ 940.175147] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 940.175147] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] raise exception.PortBindingFailed(port_id=port['id']) [ 940.175147] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] nova.exception.PortBindingFailed: Binding failed for port 461e9d84-836a-4f8b-8ea2-f4ea9f208eef, please check neutron logs for more information. [ 940.175147] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] [ 940.175147] env[62499]: INFO nova.compute.manager [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Terminating instance [ 940.176201] env[62499]: DEBUG nova.network.neutron [req-c94b552e-fadd-48be-88ab-d7fb4e8fcb84 req-4f488584-12ac-4276-8205-9ce9a0e9d764 service nova] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 940.178038] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquiring lock "refresh_cache-99920efd-13e6-4a18-9532-4b53bea3c736" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 940.249139] env[62499]: DEBUG nova.network.neutron [req-c94b552e-fadd-48be-88ab-d7fb4e8fcb84 req-4f488584-12ac-4276-8205-9ce9a0e9d764 service nova] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 940.342949] env[62499]: DEBUG nova.network.neutron [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 940.421245] env[62499]: DEBUG nova.network.neutron [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 940.461189] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72b34707-0263-4e3d-94e9-412693b98540 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 940.469044] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79244cb8-b959-4695-8adb-928fbe5c8a52 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 940.496894] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-acd41745-3b36-47cd-a6b6-077cb02efdf4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 940.503709] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86077ad8-585f-4e5a-a41f-c334639776e5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 940.516078] env[62499]: DEBUG nova.compute.provider_tree [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 940.752553] env[62499]: DEBUG oslo_concurrency.lockutils [req-c94b552e-fadd-48be-88ab-d7fb4e8fcb84 req-4f488584-12ac-4276-8205-9ce9a0e9d764 service nova] Releasing lock "refresh_cache-99920efd-13e6-4a18-9532-4b53bea3c736" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 940.753086] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquired lock "refresh_cache-99920efd-13e6-4a18-9532-4b53bea3c736" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 940.753262] env[62499]: DEBUG nova.network.neutron [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 940.925836] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Releasing lock "refresh_cache-f1b5512b-702f-45b1-9844-13a1218c31b3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 940.926118] env[62499]: DEBUG nova.compute.manager [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 940.926394] env[62499]: DEBUG nova.compute.manager [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 940.926463] env[62499]: DEBUG nova.network.neutron [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 940.941962] env[62499]: DEBUG nova.network.neutron [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 941.018812] env[62499]: DEBUG nova.scheduler.client.report [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 941.271888] env[62499]: DEBUG nova.network.neutron [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 941.341941] env[62499]: DEBUG nova.network.neutron [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 941.445523] env[62499]: DEBUG nova.network.neutron [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 941.523511] env[62499]: DEBUG oslo_concurrency.lockutils [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.704s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 941.525728] env[62499]: DEBUG oslo_concurrency.lockutils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.037s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 941.541652] env[62499]: INFO nova.scheduler.client.report [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Deleted allocations for instance b534fcdb-b0db-4e07-85fc-696dc58168aa [ 941.679290] env[62499]: DEBUG nova.compute.manager [req-07c80a9e-b05a-4c06-a7b2-b8a04c0dfdbb req-a9fca6b3-cdad-46a9-b50a-5072cbb46ce4 service nova] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Received event network-vif-deleted-461e9d84-836a-4f8b-8ea2-f4ea9f208eef {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 941.844530] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Releasing lock "refresh_cache-99920efd-13e6-4a18-9532-4b53bea3c736" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 941.844991] env[62499]: DEBUG nova.compute.manager [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 941.845203] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 941.845493] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-89d573f0-1947-4d76-8e4b-4c840e72021d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 941.854909] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f3cf642d-8aa4-4ef7-a598-5f078e017270 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 941.877664] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 99920efd-13e6-4a18-9532-4b53bea3c736 could not be found. [ 941.877935] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 941.878177] env[62499]: INFO nova.compute.manager [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Took 0.03 seconds to destroy the instance on the hypervisor. [ 941.878458] env[62499]: DEBUG oslo.service.loopingcall [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 941.878703] env[62499]: DEBUG nova.compute.manager [-] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 941.878833] env[62499]: DEBUG nova.network.neutron [-] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 941.894526] env[62499]: DEBUG nova.network.neutron [-] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 941.947366] env[62499]: INFO nova.compute.manager [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] [instance: f1b5512b-702f-45b1-9844-13a1218c31b3] Took 1.02 seconds to deallocate network for instance. [ 942.054268] env[62499]: DEBUG oslo_concurrency.lockutils [None req-322467c3-794c-4d9d-8a74-a90d841b21a6 tempest-ServerShowV257Test-1145004636 tempest-ServerShowV257Test-1145004636-project-member] Lock "b534fcdb-b0db-4e07-85fc-696dc58168aa" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 20.360s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 942.165767] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02c44163-b828-45b5-bbb6-69a0a0f158de {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 942.173631] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f3d9434e-7407-4627-ac1c-9a0053e104f7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 942.203633] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1fefdc02-30b7-4329-9a4c-97b527befc45 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 942.210548] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7683244d-b256-4544-95f5-99fbb9261f37 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 942.223591] env[62499]: DEBUG nova.compute.provider_tree [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 942.397087] env[62499]: DEBUG nova.network.neutron [-] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 942.727357] env[62499]: DEBUG nova.scheduler.client.report [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 942.900130] env[62499]: INFO nova.compute.manager [-] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Took 1.02 seconds to deallocate network for instance. [ 942.902975] env[62499]: DEBUG nova.compute.claims [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 942.903171] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 942.974807] env[62499]: INFO nova.scheduler.client.report [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Deleted allocations for instance f1b5512b-702f-45b1-9844-13a1218c31b3 [ 943.234699] env[62499]: DEBUG oslo_concurrency.lockutils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.709s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 943.235356] env[62499]: ERROR nova.compute.manager [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 21aa756e-d6a2-47b0-92ea-2b12642e43a1, please check neutron logs for more information. [ 943.235356] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Traceback (most recent call last): [ 943.235356] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 943.235356] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] self.driver.spawn(context, instance, image_meta, [ 943.235356] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 943.235356] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] self._vmops.spawn(context, instance, image_meta, injected_files, [ 943.235356] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 943.235356] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] vm_ref = self.build_virtual_machine(instance, [ 943.235356] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 943.235356] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] vif_infos = vmwarevif.get_vif_info(self._session, [ 943.235356] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 943.235708] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] for vif in network_info: [ 943.235708] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 943.235708] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] return self._sync_wrapper(fn, *args, **kwargs) [ 943.235708] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 943.235708] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] self.wait() [ 943.235708] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 943.235708] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] self[:] = self._gt.wait() [ 943.235708] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 943.235708] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] return self._exit_event.wait() [ 943.235708] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 943.235708] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] current.throw(*self._exc) [ 943.235708] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 943.235708] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] result = function(*args, **kwargs) [ 943.236151] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 943.236151] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] return func(*args, **kwargs) [ 943.236151] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 943.236151] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] raise e [ 943.236151] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 943.236151] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] nwinfo = self.network_api.allocate_for_instance( [ 943.236151] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 943.236151] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] created_port_ids = self._update_ports_for_instance( [ 943.236151] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 943.236151] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] with excutils.save_and_reraise_exception(): [ 943.236151] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 943.236151] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] self.force_reraise() [ 943.236151] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 943.236530] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] raise self.value [ 943.236530] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 943.236530] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] updated_port = self._update_port( [ 943.236530] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 943.236530] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] _ensure_no_port_binding_failure(port) [ 943.236530] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 943.236530] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] raise exception.PortBindingFailed(port_id=port['id']) [ 943.236530] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] nova.exception.PortBindingFailed: Binding failed for port 21aa756e-d6a2-47b0-92ea-2b12642e43a1, please check neutron logs for more information. [ 943.236530] env[62499]: ERROR nova.compute.manager [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] [ 943.236530] env[62499]: DEBUG nova.compute.utils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Binding failed for port 21aa756e-d6a2-47b0-92ea-2b12642e43a1, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 943.237620] env[62499]: DEBUG oslo_concurrency.lockutils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 12.749s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 943.239082] env[62499]: INFO nova.compute.claims [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 943.242276] env[62499]: DEBUG nova.compute.manager [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Build of instance 4af6c7f4-b774-4c3f-80af-e8704a904c38 was re-scheduled: Binding failed for port 21aa756e-d6a2-47b0-92ea-2b12642e43a1, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 943.242682] env[62499]: DEBUG nova.compute.manager [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 943.242903] env[62499]: DEBUG oslo_concurrency.lockutils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Acquiring lock "refresh_cache-4af6c7f4-b774-4c3f-80af-e8704a904c38" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 943.243060] env[62499]: DEBUG oslo_concurrency.lockutils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Acquired lock "refresh_cache-4af6c7f4-b774-4c3f-80af-e8704a904c38" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 943.243222] env[62499]: DEBUG nova.network.neutron [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 943.482617] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1ee7959d-e417-4788-8ce5-5e53e9740263 tempest-AttachInterfacesUnderV243Test-220457506 tempest-AttachInterfacesUnderV243Test-220457506-project-member] Lock "f1b5512b-702f-45b1-9844-13a1218c31b3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 77.665s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 943.770495] env[62499]: DEBUG nova.network.neutron [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 943.905990] env[62499]: DEBUG nova.network.neutron [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 944.365127] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-580f9f89-7239-43f1-accf-87156f54d110 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 944.373039] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-403934b1-504c-4cb0-92ab-465147607136 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 944.404060] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9002d029-cb08-46db-b5d3-47abea4be358 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 944.408176] env[62499]: DEBUG oslo_concurrency.lockutils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Releasing lock "refresh_cache-4af6c7f4-b774-4c3f-80af-e8704a904c38" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 944.408394] env[62499]: DEBUG nova.compute.manager [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 944.408569] env[62499]: DEBUG nova.compute.manager [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 944.408733] env[62499]: DEBUG nova.network.neutron [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 944.413262] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e14c93b-33e5-4ce2-9354-5b36d9ebc941 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 944.425788] env[62499]: DEBUG nova.compute.provider_tree [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 944.431268] env[62499]: DEBUG nova.network.neutron [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 944.934541] env[62499]: DEBUG nova.scheduler.client.report [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 944.938025] env[62499]: DEBUG nova.network.neutron [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 945.440429] env[62499]: DEBUG oslo_concurrency.lockutils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.203s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 945.441027] env[62499]: DEBUG nova.compute.manager [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 945.443624] env[62499]: INFO nova.compute.manager [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] [instance: 4af6c7f4-b774-4c3f-80af-e8704a904c38] Took 1.03 seconds to deallocate network for instance. [ 945.446392] env[62499]: DEBUG oslo_concurrency.lockutils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.671s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 946.032765] env[62499]: DEBUG nova.compute.utils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 946.041746] env[62499]: DEBUG nova.compute.manager [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 946.042446] env[62499]: DEBUG nova.network.neutron [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 946.101386] env[62499]: DEBUG nova.policy [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '07524734e344412e9ddc0904ac1e135f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'a7226f416d264f4e8888c523883bd131', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 946.191169] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c65ad1b5-b597-4b27-8cf7-8c35dacd43e8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 946.198771] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d522931-8f08-4ec4-a0ab-18d8486301c5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 946.228743] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da0ec9c6-5133-4551-8770-bdb01723d74d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 946.236028] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64613387-290a-4f62-91b5-53428b48d04f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 946.248717] env[62499]: DEBUG nova.compute.provider_tree [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 946.395024] env[62499]: DEBUG nova.network.neutron [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Successfully created port: 5f7a9791-8f2a-445c-a05f-00d4c5bd13c5 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 946.544880] env[62499]: DEBUG nova.compute.manager [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 946.578261] env[62499]: INFO nova.scheduler.client.report [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Deleted allocations for instance 4af6c7f4-b774-4c3f-80af-e8704a904c38 [ 946.754016] env[62499]: DEBUG nova.scheduler.client.report [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 947.086150] env[62499]: DEBUG oslo_concurrency.lockutils [None req-91d05791-7744-4fb6-b96a-b6945b71df02 tempest-InstanceActionsV221TestJSON-464989833 tempest-InstanceActionsV221TestJSON-464989833-project-member] Lock "4af6c7f4-b774-4c3f-80af-e8704a904c38" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 72.844s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 947.256016] env[62499]: DEBUG nova.compute.manager [req-beb00451-dd98-4384-85a9-73201894d72d req-c49c12fc-30eb-4026-a17b-49d7d531480c service nova] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Received event network-changed-5f7a9791-8f2a-445c-a05f-00d4c5bd13c5 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 947.256373] env[62499]: DEBUG nova.compute.manager [req-beb00451-dd98-4384-85a9-73201894d72d req-c49c12fc-30eb-4026-a17b-49d7d531480c service nova] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Refreshing instance network info cache due to event network-changed-5f7a9791-8f2a-445c-a05f-00d4c5bd13c5. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 947.256642] env[62499]: DEBUG oslo_concurrency.lockutils [req-beb00451-dd98-4384-85a9-73201894d72d req-c49c12fc-30eb-4026-a17b-49d7d531480c service nova] Acquiring lock "refresh_cache-4170a2f4-73ff-4bc6-9dc6-d2c4765c449d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 947.256792] env[62499]: DEBUG oslo_concurrency.lockutils [req-beb00451-dd98-4384-85a9-73201894d72d req-c49c12fc-30eb-4026-a17b-49d7d531480c service nova] Acquired lock "refresh_cache-4170a2f4-73ff-4bc6-9dc6-d2c4765c449d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 947.257867] env[62499]: DEBUG nova.network.neutron [req-beb00451-dd98-4384-85a9-73201894d72d req-c49c12fc-30eb-4026-a17b-49d7d531480c service nova] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Refreshing network info cache for port 5f7a9791-8f2a-445c-a05f-00d4c5bd13c5 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 947.259900] env[62499]: DEBUG oslo_concurrency.lockutils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.814s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 947.260502] env[62499]: ERROR nova.compute.manager [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d7f7e92e-10d8-46bd-8a94-a1d88ba5bbdd, please check neutron logs for more information. [ 947.260502] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Traceback (most recent call last): [ 947.260502] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 947.260502] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] self.driver.spawn(context, instance, image_meta, [ 947.260502] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 947.260502] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 947.260502] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 947.260502] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] vm_ref = self.build_virtual_machine(instance, [ 947.260502] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 947.260502] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] vif_infos = vmwarevif.get_vif_info(self._session, [ 947.260502] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 947.260830] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] for vif in network_info: [ 947.260830] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 947.260830] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] return self._sync_wrapper(fn, *args, **kwargs) [ 947.260830] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 947.260830] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] self.wait() [ 947.260830] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 947.260830] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] self[:] = self._gt.wait() [ 947.260830] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 947.260830] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] return self._exit_event.wait() [ 947.260830] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 947.260830] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] result = hub.switch() [ 947.260830] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 947.260830] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] return self.greenlet.switch() [ 947.261171] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 947.261171] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] result = function(*args, **kwargs) [ 947.261171] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 947.261171] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] return func(*args, **kwargs) [ 947.261171] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 947.261171] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] raise e [ 947.261171] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 947.261171] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] nwinfo = self.network_api.allocate_for_instance( [ 947.261171] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 947.261171] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] created_port_ids = self._update_ports_for_instance( [ 947.261171] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 947.261171] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] with excutils.save_and_reraise_exception(): [ 947.261171] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 947.261504] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] self.force_reraise() [ 947.261504] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 947.261504] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] raise self.value [ 947.261504] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 947.261504] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] updated_port = self._update_port( [ 947.261504] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 947.261504] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] _ensure_no_port_binding_failure(port) [ 947.261504] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 947.261504] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] raise exception.PortBindingFailed(port_id=port['id']) [ 947.261504] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] nova.exception.PortBindingFailed: Binding failed for port d7f7e92e-10d8-46bd-8a94-a1d88ba5bbdd, please check neutron logs for more information. [ 947.261504] env[62499]: ERROR nova.compute.manager [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] [ 947.261913] env[62499]: DEBUG nova.compute.utils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Binding failed for port d7f7e92e-10d8-46bd-8a94-a1d88ba5bbdd, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 947.263166] env[62499]: DEBUG nova.compute.manager [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Build of instance 4fa63411-a721-4e34-a08d-90587b0b34d3 was re-scheduled: Binding failed for port d7f7e92e-10d8-46bd-8a94-a1d88ba5bbdd, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 947.263593] env[62499]: DEBUG nova.compute.manager [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 947.263805] env[62499]: DEBUG oslo_concurrency.lockutils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "refresh_cache-4fa63411-a721-4e34-a08d-90587b0b34d3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 947.264043] env[62499]: DEBUG oslo_concurrency.lockutils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquired lock "refresh_cache-4fa63411-a721-4e34-a08d-90587b0b34d3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 947.264342] env[62499]: DEBUG nova.network.neutron [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 947.269027] env[62499]: DEBUG oslo_concurrency.lockutils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.918s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 947.411719] env[62499]: ERROR nova.compute.manager [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 5f7a9791-8f2a-445c-a05f-00d4c5bd13c5, please check neutron logs for more information. [ 947.411719] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 947.411719] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 947.411719] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 947.411719] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 947.411719] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 947.411719] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 947.411719] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 947.411719] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 947.411719] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 947.411719] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 947.411719] env[62499]: ERROR nova.compute.manager raise self.value [ 947.411719] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 947.411719] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 947.411719] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 947.411719] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 947.412209] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 947.412209] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 947.412209] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 5f7a9791-8f2a-445c-a05f-00d4c5bd13c5, please check neutron logs for more information. [ 947.412209] env[62499]: ERROR nova.compute.manager [ 947.412209] env[62499]: Traceback (most recent call last): [ 947.412209] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 947.412209] env[62499]: listener.cb(fileno) [ 947.412209] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 947.412209] env[62499]: result = function(*args, **kwargs) [ 947.412209] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 947.412209] env[62499]: return func(*args, **kwargs) [ 947.412209] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 947.412209] env[62499]: raise e [ 947.412209] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 947.412209] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 947.412209] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 947.412209] env[62499]: created_port_ids = self._update_ports_for_instance( [ 947.412209] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 947.412209] env[62499]: with excutils.save_and_reraise_exception(): [ 947.412209] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 947.412209] env[62499]: self.force_reraise() [ 947.412209] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 947.412209] env[62499]: raise self.value [ 947.412209] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 947.412209] env[62499]: updated_port = self._update_port( [ 947.412209] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 947.412209] env[62499]: _ensure_no_port_binding_failure(port) [ 947.412209] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 947.412209] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 947.414692] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 5f7a9791-8f2a-445c-a05f-00d4c5bd13c5, please check neutron logs for more information. [ 947.414692] env[62499]: Removing descriptor: 17 [ 947.558727] env[62499]: DEBUG nova.compute.manager [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 947.589596] env[62499]: DEBUG nova.virt.hardware [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 947.589838] env[62499]: DEBUG nova.virt.hardware [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 947.590065] env[62499]: DEBUG nova.virt.hardware [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 947.590411] env[62499]: DEBUG nova.virt.hardware [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 947.590602] env[62499]: DEBUG nova.virt.hardware [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 947.590775] env[62499]: DEBUG nova.virt.hardware [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 947.591008] env[62499]: DEBUG nova.virt.hardware [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 947.591208] env[62499]: DEBUG nova.virt.hardware [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 947.591382] env[62499]: DEBUG nova.virt.hardware [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 947.591541] env[62499]: DEBUG nova.virt.hardware [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 947.591708] env[62499]: DEBUG nova.virt.hardware [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 947.592553] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93956524-6aac-4b40-9bb0-2ebf7e80d873 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.600783] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-278a7564-ff49-46ca-a432-38230d204b49 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.614013] env[62499]: ERROR nova.compute.manager [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 5f7a9791-8f2a-445c-a05f-00d4c5bd13c5, please check neutron logs for more information. [ 947.614013] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Traceback (most recent call last): [ 947.614013] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 947.614013] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] yield resources [ 947.614013] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 947.614013] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] self.driver.spawn(context, instance, image_meta, [ 947.614013] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 947.614013] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 947.614013] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 947.614013] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] vm_ref = self.build_virtual_machine(instance, [ 947.614013] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 947.614373] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] vif_infos = vmwarevif.get_vif_info(self._session, [ 947.614373] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 947.614373] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] for vif in network_info: [ 947.614373] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 947.614373] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] return self._sync_wrapper(fn, *args, **kwargs) [ 947.614373] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 947.614373] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] self.wait() [ 947.614373] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 947.614373] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] self[:] = self._gt.wait() [ 947.614373] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 947.614373] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] return self._exit_event.wait() [ 947.614373] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 947.614373] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] current.throw(*self._exc) [ 947.614750] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 947.614750] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] result = function(*args, **kwargs) [ 947.614750] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 947.614750] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] return func(*args, **kwargs) [ 947.614750] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 947.614750] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] raise e [ 947.614750] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 947.614750] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] nwinfo = self.network_api.allocate_for_instance( [ 947.614750] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 947.614750] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] created_port_ids = self._update_ports_for_instance( [ 947.614750] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 947.614750] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] with excutils.save_and_reraise_exception(): [ 947.614750] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 947.615379] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] self.force_reraise() [ 947.615379] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 947.615379] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] raise self.value [ 947.615379] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 947.615379] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] updated_port = self._update_port( [ 947.615379] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 947.615379] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] _ensure_no_port_binding_failure(port) [ 947.615379] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 947.615379] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] raise exception.PortBindingFailed(port_id=port['id']) [ 947.615379] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] nova.exception.PortBindingFailed: Binding failed for port 5f7a9791-8f2a-445c-a05f-00d4c5bd13c5, please check neutron logs for more information. [ 947.615379] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] [ 947.615379] env[62499]: INFO nova.compute.manager [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Terminating instance [ 947.616477] env[62499]: DEBUG oslo_concurrency.lockutils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Acquiring lock "refresh_cache-4170a2f4-73ff-4bc6-9dc6-d2c4765c449d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 947.779752] env[62499]: DEBUG nova.network.neutron [req-beb00451-dd98-4384-85a9-73201894d72d req-c49c12fc-30eb-4026-a17b-49d7d531480c service nova] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 947.791011] env[62499]: DEBUG nova.network.neutron [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 947.863418] env[62499]: DEBUG nova.network.neutron [req-beb00451-dd98-4384-85a9-73201894d72d req-c49c12fc-30eb-4026-a17b-49d7d531480c service nova] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 947.892196] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6526ec41-45ad-445a-9feb-d460b3432824 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.901011] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d17834f-78bb-4309-8b02-c20bed7e41e4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.940363] env[62499]: DEBUG nova.network.neutron [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 947.942105] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-46050a34-35ba-4619-a1c1-829ac8cb0cfd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.950546] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3f57719-9695-4c6d-9c27-f0c443235e13 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.964924] env[62499]: DEBUG nova.compute.provider_tree [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 948.366405] env[62499]: DEBUG oslo_concurrency.lockutils [req-beb00451-dd98-4384-85a9-73201894d72d req-c49c12fc-30eb-4026-a17b-49d7d531480c service nova] Releasing lock "refresh_cache-4170a2f4-73ff-4bc6-9dc6-d2c4765c449d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 948.366880] env[62499]: DEBUG oslo_concurrency.lockutils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Acquired lock "refresh_cache-4170a2f4-73ff-4bc6-9dc6-d2c4765c449d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 948.367117] env[62499]: DEBUG nova.network.neutron [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 948.451544] env[62499]: DEBUG oslo_concurrency.lockutils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Releasing lock "refresh_cache-4fa63411-a721-4e34-a08d-90587b0b34d3" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 948.453776] env[62499]: DEBUG nova.compute.manager [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 948.454379] env[62499]: DEBUG nova.compute.manager [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 948.454379] env[62499]: DEBUG nova.network.neutron [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 948.468643] env[62499]: DEBUG nova.scheduler.client.report [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 948.478557] env[62499]: DEBUG nova.network.neutron [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 948.884069] env[62499]: DEBUG nova.network.neutron [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 948.968653] env[62499]: DEBUG nova.network.neutron [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 948.974132] env[62499]: DEBUG oslo_concurrency.lockutils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.706s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 948.974725] env[62499]: ERROR nova.compute.manager [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 17a49625-1200-4a66-81d3-8a6e144d9234, please check neutron logs for more information. [ 948.974725] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] Traceback (most recent call last): [ 948.974725] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 948.974725] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] self.driver.spawn(context, instance, image_meta, [ 948.974725] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 948.974725] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] self._vmops.spawn(context, instance, image_meta, injected_files, [ 948.974725] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 948.974725] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] vm_ref = self.build_virtual_machine(instance, [ 948.974725] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 948.974725] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] vif_infos = vmwarevif.get_vif_info(self._session, [ 948.974725] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 948.975301] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] for vif in network_info: [ 948.975301] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 948.975301] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] return self._sync_wrapper(fn, *args, **kwargs) [ 948.975301] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 948.975301] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] self.wait() [ 948.975301] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 948.975301] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] self[:] = self._gt.wait() [ 948.975301] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 948.975301] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] return self._exit_event.wait() [ 948.975301] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 948.975301] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] current.throw(*self._exc) [ 948.975301] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 948.975301] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] result = function(*args, **kwargs) [ 948.975735] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 948.975735] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] return func(*args, **kwargs) [ 948.975735] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 948.975735] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] raise e [ 948.975735] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 948.975735] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] nwinfo = self.network_api.allocate_for_instance( [ 948.975735] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 948.975735] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] created_port_ids = self._update_ports_for_instance( [ 948.975735] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 948.975735] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] with excutils.save_and_reraise_exception(): [ 948.975735] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 948.975735] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] self.force_reraise() [ 948.975735] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 948.976121] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] raise self.value [ 948.976121] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 948.976121] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] updated_port = self._update_port( [ 948.976121] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 948.976121] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] _ensure_no_port_binding_failure(port) [ 948.976121] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 948.976121] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] raise exception.PortBindingFailed(port_id=port['id']) [ 948.976121] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] nova.exception.PortBindingFailed: Binding failed for port 17a49625-1200-4a66-81d3-8a6e144d9234, please check neutron logs for more information. [ 948.976121] env[62499]: ERROR nova.compute.manager [instance: 88466440-7392-43d3-83ec-45229516a3ef] [ 948.976121] env[62499]: DEBUG nova.compute.utils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Binding failed for port 17a49625-1200-4a66-81d3-8a6e144d9234, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 948.976510] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.462s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 948.977900] env[62499]: INFO nova.compute.claims [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 948.980743] env[62499]: DEBUG nova.compute.manager [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Build of instance 88466440-7392-43d3-83ec-45229516a3ef was re-scheduled: Binding failed for port 17a49625-1200-4a66-81d3-8a6e144d9234, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 948.981189] env[62499]: DEBUG nova.compute.manager [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 948.981405] env[62499]: DEBUG oslo_concurrency.lockutils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Acquiring lock "refresh_cache-88466440-7392-43d3-83ec-45229516a3ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 948.981546] env[62499]: DEBUG oslo_concurrency.lockutils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Acquired lock "refresh_cache-88466440-7392-43d3-83ec-45229516a3ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 948.981698] env[62499]: DEBUG nova.network.neutron [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 948.982895] env[62499]: DEBUG nova.network.neutron [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 949.280455] env[62499]: DEBUG nova.compute.manager [req-b4fabc1c-73a9-4c99-8aee-349fd560ad3e req-b21062fc-a915-4c80-a563-2978fedf5380 service nova] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Received event network-vif-deleted-5f7a9791-8f2a-445c-a05f-00d4c5bd13c5 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 949.471205] env[62499]: DEBUG oslo_concurrency.lockutils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Releasing lock "refresh_cache-4170a2f4-73ff-4bc6-9dc6-d2c4765c449d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 949.471701] env[62499]: DEBUG nova.compute.manager [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 949.471904] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 949.472236] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e8d195b9-3002-4775-acfd-913ff0e67b09 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 949.482603] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2826531-1721-42dd-9635-941e8700e25b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 949.493034] env[62499]: INFO nova.compute.manager [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: 4fa63411-a721-4e34-a08d-90587b0b34d3] Took 1.04 seconds to deallocate network for instance. [ 949.510524] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d could not be found. [ 949.510742] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 949.510918] env[62499]: INFO nova.compute.manager [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Took 0.04 seconds to destroy the instance on the hypervisor. [ 949.511193] env[62499]: DEBUG oslo.service.loopingcall [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 949.511444] env[62499]: DEBUG nova.compute.manager [-] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 949.511537] env[62499]: DEBUG nova.network.neutron [-] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 949.515018] env[62499]: DEBUG nova.network.neutron [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 949.530497] env[62499]: DEBUG nova.network.neutron [-] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 949.592560] env[62499]: DEBUG nova.network.neutron [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 950.033867] env[62499]: DEBUG nova.network.neutron [-] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 950.097518] env[62499]: DEBUG oslo_concurrency.lockutils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Releasing lock "refresh_cache-88466440-7392-43d3-83ec-45229516a3ef" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 950.097518] env[62499]: DEBUG nova.compute.manager [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 950.097518] env[62499]: DEBUG nova.compute.manager [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 950.097518] env[62499]: DEBUG nova.network.neutron [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 950.114246] env[62499]: DEBUG nova.network.neutron [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 950.119750] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6643528d-62a7-474e-87b7-36dab7313da1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.128044] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc50ef74-223d-4f90-960f-0dc260b2f0f7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.165408] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e4e95e8d-90b1-4337-b485-474e1e9af399 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.174279] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d7bc409-3a6c-4765-a295-5f317e09da36 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.188504] env[62499]: DEBUG nova.compute.provider_tree [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 950.532522] env[62499]: INFO nova.scheduler.client.report [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Deleted allocations for instance 4fa63411-a721-4e34-a08d-90587b0b34d3 [ 950.538895] env[62499]: INFO nova.compute.manager [-] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Took 1.03 seconds to deallocate network for instance. [ 950.545448] env[62499]: DEBUG nova.compute.claims [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 950.545663] env[62499]: DEBUG oslo_concurrency.lockutils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 950.616884] env[62499]: DEBUG nova.network.neutron [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 950.696317] env[62499]: DEBUG nova.scheduler.client.report [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 951.047722] env[62499]: DEBUG oslo_concurrency.lockutils [None req-184b2e2f-40aa-49c1-a157-298cc02d5fbc tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "4fa63411-a721-4e34-a08d-90587b0b34d3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 76.089s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 951.120587] env[62499]: INFO nova.compute.manager [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] [instance: 88466440-7392-43d3-83ec-45229516a3ef] Took 1.02 seconds to deallocate network for instance. [ 951.203525] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.226s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 951.203598] env[62499]: DEBUG nova.compute.manager [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 951.206960] env[62499]: DEBUG oslo_concurrency.lockutils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.622s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 951.708749] env[62499]: DEBUG nova.compute.utils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 951.711206] env[62499]: DEBUG nova.compute.manager [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 951.711403] env[62499]: DEBUG nova.network.neutron [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 951.801659] env[62499]: DEBUG nova.policy [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '02640299eb9c4a65bc80de6242dd2cd5', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f509dfe537d441bcacb1a4aab3607214', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 951.856028] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d962e8bc-ee73-440a-9336-a7ef54b403e4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 951.865325] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0f7ea19-ec5e-47d5-807b-6213a62f7860 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 951.901877] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dfc62a92-c5c2-4bb2-8a07-952b39e924b6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 951.910022] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5889a826-ade4-4f35-879b-31387c9d7178 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 951.922911] env[62499]: DEBUG nova.compute.provider_tree [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 952.093722] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Acquiring lock "636d27e5-f184-4656-9225-09c6ec0f0772" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 952.093964] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Lock "636d27e5-f184-4656-9225-09c6ec0f0772" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 952.095422] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "b4e185e6-b177-4212-bad8-b9fd1b51ea95" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 952.095422] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "b4e185e6-b177-4212-bad8-b9fd1b51ea95" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 952.120277] env[62499]: DEBUG nova.network.neutron [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Successfully created port: 438d8dc0-9244-438c-b078-fa7754182d12 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 952.152881] env[62499]: INFO nova.scheduler.client.report [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Deleted allocations for instance 88466440-7392-43d3-83ec-45229516a3ef [ 952.215713] env[62499]: DEBUG nova.compute.manager [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 952.427280] env[62499]: DEBUG nova.scheduler.client.report [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 952.596880] env[62499]: DEBUG nova.compute.manager [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 952.599571] env[62499]: DEBUG nova.compute.manager [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 952.667552] env[62499]: DEBUG oslo_concurrency.lockutils [None req-12f3fb3d-058f-4625-8df5-6b9ab03b1e0c tempest-ServerMetadataNegativeTestJSON-1679020876 tempest-ServerMetadataNegativeTestJSON-1679020876-project-member] Lock "88466440-7392-43d3-83ec-45229516a3ef" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 75.947s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 952.842072] env[62499]: DEBUG nova.compute.manager [req-487ece8f-5aee-4f87-998c-8b1b36fd1e68 req-7d708ad8-f535-45ef-aa46-844770bd168a service nova] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Received event network-changed-438d8dc0-9244-438c-b078-fa7754182d12 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 952.842350] env[62499]: DEBUG nova.compute.manager [req-487ece8f-5aee-4f87-998c-8b1b36fd1e68 req-7d708ad8-f535-45ef-aa46-844770bd168a service nova] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Refreshing instance network info cache due to event network-changed-438d8dc0-9244-438c-b078-fa7754182d12. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 952.842490] env[62499]: DEBUG oslo_concurrency.lockutils [req-487ece8f-5aee-4f87-998c-8b1b36fd1e68 req-7d708ad8-f535-45ef-aa46-844770bd168a service nova] Acquiring lock "refresh_cache-ca937041-1aa6-46d8-be4e-b4983652fe15" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 952.842630] env[62499]: DEBUG oslo_concurrency.lockutils [req-487ece8f-5aee-4f87-998c-8b1b36fd1e68 req-7d708ad8-f535-45ef-aa46-844770bd168a service nova] Acquired lock "refresh_cache-ca937041-1aa6-46d8-be4e-b4983652fe15" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 952.842782] env[62499]: DEBUG nova.network.neutron [req-487ece8f-5aee-4f87-998c-8b1b36fd1e68 req-7d708ad8-f535-45ef-aa46-844770bd168a service nova] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Refreshing network info cache for port 438d8dc0-9244-438c-b078-fa7754182d12 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 952.932462] env[62499]: DEBUG oslo_concurrency.lockutils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.725s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 952.933044] env[62499]: ERROR nova.compute.manager [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d984b948-6a63-4e30-b2af-b83378f4d62b, please check neutron logs for more information. [ 952.933044] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Traceback (most recent call last): [ 952.933044] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 952.933044] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] self.driver.spawn(context, instance, image_meta, [ 952.933044] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 952.933044] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] self._vmops.spawn(context, instance, image_meta, injected_files, [ 952.933044] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 952.933044] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] vm_ref = self.build_virtual_machine(instance, [ 952.933044] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 952.933044] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] vif_infos = vmwarevif.get_vif_info(self._session, [ 952.933044] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 952.933536] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] for vif in network_info: [ 952.933536] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 952.933536] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] return self._sync_wrapper(fn, *args, **kwargs) [ 952.933536] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 952.933536] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] self.wait() [ 952.933536] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 952.933536] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] self[:] = self._gt.wait() [ 952.933536] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 952.933536] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] return self._exit_event.wait() [ 952.933536] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 952.933536] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] current.throw(*self._exc) [ 952.933536] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 952.933536] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] result = function(*args, **kwargs) [ 952.933964] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 952.933964] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] return func(*args, **kwargs) [ 952.933964] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 952.933964] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] raise e [ 952.933964] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 952.933964] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] nwinfo = self.network_api.allocate_for_instance( [ 952.933964] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 952.933964] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] created_port_ids = self._update_ports_for_instance( [ 952.933964] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 952.933964] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] with excutils.save_and_reraise_exception(): [ 952.933964] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 952.933964] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] self.force_reraise() [ 952.933964] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 952.934420] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] raise self.value [ 952.934420] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 952.934420] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] updated_port = self._update_port( [ 952.934420] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 952.934420] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] _ensure_no_port_binding_failure(port) [ 952.934420] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 952.934420] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] raise exception.PortBindingFailed(port_id=port['id']) [ 952.934420] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] nova.exception.PortBindingFailed: Binding failed for port d984b948-6a63-4e30-b2af-b83378f4d62b, please check neutron logs for more information. [ 952.934420] env[62499]: ERROR nova.compute.manager [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] [ 952.934420] env[62499]: DEBUG nova.compute.utils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Binding failed for port d984b948-6a63-4e30-b2af-b83378f4d62b, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 952.935012] env[62499]: DEBUG oslo_concurrency.lockutils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.085s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 952.942689] env[62499]: DEBUG nova.compute.manager [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Build of instance 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87 was re-scheduled: Binding failed for port d984b948-6a63-4e30-b2af-b83378f4d62b, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 952.944919] env[62499]: DEBUG nova.compute.manager [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 952.944919] env[62499]: DEBUG oslo_concurrency.lockutils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Acquiring lock "refresh_cache-95a5cb2c-24da-43c0-b9d1-7f0b51ccda87" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 952.944919] env[62499]: DEBUG oslo_concurrency.lockutils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Acquired lock "refresh_cache-95a5cb2c-24da-43c0-b9d1-7f0b51ccda87" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 952.944919] env[62499]: DEBUG nova.network.neutron [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 953.022350] env[62499]: ERROR nova.compute.manager [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 438d8dc0-9244-438c-b078-fa7754182d12, please check neutron logs for more information. [ 953.022350] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 953.022350] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 953.022350] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 953.022350] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 953.022350] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 953.022350] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 953.022350] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 953.022350] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 953.022350] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 953.022350] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 953.022350] env[62499]: ERROR nova.compute.manager raise self.value [ 953.022350] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 953.022350] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 953.022350] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 953.022350] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 953.022888] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 953.022888] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 953.022888] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 438d8dc0-9244-438c-b078-fa7754182d12, please check neutron logs for more information. [ 953.022888] env[62499]: ERROR nova.compute.manager [ 953.022888] env[62499]: Traceback (most recent call last): [ 953.022888] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 953.022888] env[62499]: listener.cb(fileno) [ 953.022888] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 953.022888] env[62499]: result = function(*args, **kwargs) [ 953.022888] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 953.022888] env[62499]: return func(*args, **kwargs) [ 953.022888] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 953.022888] env[62499]: raise e [ 953.022888] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 953.022888] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 953.022888] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 953.022888] env[62499]: created_port_ids = self._update_ports_for_instance( [ 953.022888] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 953.022888] env[62499]: with excutils.save_and_reraise_exception(): [ 953.022888] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 953.022888] env[62499]: self.force_reraise() [ 953.022888] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 953.022888] env[62499]: raise self.value [ 953.022888] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 953.022888] env[62499]: updated_port = self._update_port( [ 953.022888] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 953.022888] env[62499]: _ensure_no_port_binding_failure(port) [ 953.022888] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 953.022888] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 953.023808] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 438d8dc0-9244-438c-b078-fa7754182d12, please check neutron logs for more information. [ 953.023808] env[62499]: Removing descriptor: 16 [ 953.129627] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 953.136084] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 953.228086] env[62499]: DEBUG nova.compute.manager [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 953.254080] env[62499]: DEBUG nova.virt.hardware [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 953.254080] env[62499]: DEBUG nova.virt.hardware [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 953.254080] env[62499]: DEBUG nova.virt.hardware [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 953.254080] env[62499]: DEBUG nova.virt.hardware [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 953.254290] env[62499]: DEBUG nova.virt.hardware [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 953.254290] env[62499]: DEBUG nova.virt.hardware [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 953.254290] env[62499]: DEBUG nova.virt.hardware [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 953.254290] env[62499]: DEBUG nova.virt.hardware [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 953.254290] env[62499]: DEBUG nova.virt.hardware [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 953.254863] env[62499]: DEBUG nova.virt.hardware [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 953.255205] env[62499]: DEBUG nova.virt.hardware [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 953.258071] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0384fa6b-27e6-4e2d-a123-f1a2bcfa6c40 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.264059] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94939eeb-c86e-49cc-ac05-8b70f745c3c2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.277970] env[62499]: ERROR nova.compute.manager [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 438d8dc0-9244-438c-b078-fa7754182d12, please check neutron logs for more information. [ 953.277970] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Traceback (most recent call last): [ 953.277970] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 953.277970] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] yield resources [ 953.277970] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 953.277970] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] self.driver.spawn(context, instance, image_meta, [ 953.277970] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 953.277970] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] self._vmops.spawn(context, instance, image_meta, injected_files, [ 953.277970] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 953.277970] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] vm_ref = self.build_virtual_machine(instance, [ 953.277970] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 953.278546] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] vif_infos = vmwarevif.get_vif_info(self._session, [ 953.278546] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 953.278546] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] for vif in network_info: [ 953.278546] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 953.278546] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] return self._sync_wrapper(fn, *args, **kwargs) [ 953.278546] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 953.278546] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] self.wait() [ 953.278546] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 953.278546] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] self[:] = self._gt.wait() [ 953.278546] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 953.278546] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] return self._exit_event.wait() [ 953.278546] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 953.278546] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] current.throw(*self._exc) [ 953.278910] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 953.278910] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] result = function(*args, **kwargs) [ 953.278910] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 953.278910] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] return func(*args, **kwargs) [ 953.278910] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 953.278910] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] raise e [ 953.278910] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 953.278910] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] nwinfo = self.network_api.allocate_for_instance( [ 953.278910] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 953.278910] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] created_port_ids = self._update_ports_for_instance( [ 953.278910] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 953.278910] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] with excutils.save_and_reraise_exception(): [ 953.278910] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 953.279292] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] self.force_reraise() [ 953.279292] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 953.279292] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] raise self.value [ 953.279292] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 953.279292] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] updated_port = self._update_port( [ 953.279292] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 953.279292] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] _ensure_no_port_binding_failure(port) [ 953.279292] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 953.279292] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] raise exception.PortBindingFailed(port_id=port['id']) [ 953.279292] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] nova.exception.PortBindingFailed: Binding failed for port 438d8dc0-9244-438c-b078-fa7754182d12, please check neutron logs for more information. [ 953.279292] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] [ 953.279292] env[62499]: INFO nova.compute.manager [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Terminating instance [ 953.280378] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "refresh_cache-ca937041-1aa6-46d8-be4e-b4983652fe15" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 953.366339] env[62499]: DEBUG nova.network.neutron [req-487ece8f-5aee-4f87-998c-8b1b36fd1e68 req-7d708ad8-f535-45ef-aa46-844770bd168a service nova] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 953.463396] env[62499]: DEBUG nova.network.neutron [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 953.493551] env[62499]: DEBUG nova.network.neutron [req-487ece8f-5aee-4f87-998c-8b1b36fd1e68 req-7d708ad8-f535-45ef-aa46-844770bd168a service nova] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 953.553476] env[62499]: DEBUG nova.network.neutron [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 953.569785] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8283e667-af5f-447f-8664-c8590b1d7dd5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.576800] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-89b51a56-d589-43ad-92d9-5c7fecdd5292 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.608168] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-322bfbc7-0c6d-4f10-9a70-f28ef79d2569 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.615931] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a901e26-9cd4-4a00-84d6-dc17256a334f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.630326] env[62499]: DEBUG nova.compute.provider_tree [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 953.997778] env[62499]: DEBUG oslo_concurrency.lockutils [req-487ece8f-5aee-4f87-998c-8b1b36fd1e68 req-7d708ad8-f535-45ef-aa46-844770bd168a service nova] Releasing lock "refresh_cache-ca937041-1aa6-46d8-be4e-b4983652fe15" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 953.998263] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquired lock "refresh_cache-ca937041-1aa6-46d8-be4e-b4983652fe15" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 953.998427] env[62499]: DEBUG nova.network.neutron [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 954.053816] env[62499]: DEBUG oslo_concurrency.lockutils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Releasing lock "refresh_cache-95a5cb2c-24da-43c0-b9d1-7f0b51ccda87" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 954.054065] env[62499]: DEBUG nova.compute.manager [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 954.054284] env[62499]: DEBUG nova.compute.manager [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 954.054472] env[62499]: DEBUG nova.network.neutron [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 954.070099] env[62499]: DEBUG nova.network.neutron [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 954.133615] env[62499]: DEBUG nova.scheduler.client.report [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 954.517078] env[62499]: DEBUG nova.network.neutron [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 954.572711] env[62499]: DEBUG nova.network.neutron [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 954.591947] env[62499]: DEBUG nova.network.neutron [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 954.638671] env[62499]: DEBUG oslo_concurrency.lockutils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.704s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 954.639324] env[62499]: ERROR nova.compute.manager [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port e5633ef6-c4da-4c0c-bf53-6f3e9160d31a, please check neutron logs for more information. [ 954.639324] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Traceback (most recent call last): [ 954.639324] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 954.639324] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] self.driver.spawn(context, instance, image_meta, [ 954.639324] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 954.639324] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] self._vmops.spawn(context, instance, image_meta, injected_files, [ 954.639324] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 954.639324] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] vm_ref = self.build_virtual_machine(instance, [ 954.639324] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 954.639324] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] vif_infos = vmwarevif.get_vif_info(self._session, [ 954.639324] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 954.639696] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] for vif in network_info: [ 954.639696] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 954.639696] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] return self._sync_wrapper(fn, *args, **kwargs) [ 954.639696] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 954.639696] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] self.wait() [ 954.639696] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 954.639696] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] self[:] = self._gt.wait() [ 954.639696] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 954.639696] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] return self._exit_event.wait() [ 954.639696] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 954.639696] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] current.throw(*self._exc) [ 954.639696] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 954.639696] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] result = function(*args, **kwargs) [ 954.640080] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 954.640080] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] return func(*args, **kwargs) [ 954.640080] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 954.640080] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] raise e [ 954.640080] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 954.640080] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] nwinfo = self.network_api.allocate_for_instance( [ 954.640080] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 954.640080] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] created_port_ids = self._update_ports_for_instance( [ 954.640080] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 954.640080] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] with excutils.save_and_reraise_exception(): [ 954.640080] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 954.640080] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] self.force_reraise() [ 954.640080] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 954.640440] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] raise self.value [ 954.640440] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 954.640440] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] updated_port = self._update_port( [ 954.640440] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 954.640440] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] _ensure_no_port_binding_failure(port) [ 954.640440] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 954.640440] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] raise exception.PortBindingFailed(port_id=port['id']) [ 954.640440] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] nova.exception.PortBindingFailed: Binding failed for port e5633ef6-c4da-4c0c-bf53-6f3e9160d31a, please check neutron logs for more information. [ 954.640440] env[62499]: ERROR nova.compute.manager [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] [ 954.640440] env[62499]: DEBUG nova.compute.utils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Binding failed for port e5633ef6-c4da-4c0c-bf53-6f3e9160d31a, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 954.641141] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.510s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 954.646337] env[62499]: DEBUG nova.compute.manager [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Build of instance 9fd7db7e-466e-4213-9e1d-210eae714884 was re-scheduled: Binding failed for port e5633ef6-c4da-4c0c-bf53-6f3e9160d31a, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 954.646778] env[62499]: DEBUG nova.compute.manager [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 954.646996] env[62499]: DEBUG oslo_concurrency.lockutils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "refresh_cache-9fd7db7e-466e-4213-9e1d-210eae714884" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 954.647231] env[62499]: DEBUG oslo_concurrency.lockutils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquired lock "refresh_cache-9fd7db7e-466e-4213-9e1d-210eae714884" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 954.647399] env[62499]: DEBUG nova.network.neutron [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 954.871880] env[62499]: DEBUG nova.compute.manager [req-aa42d823-2930-4411-8418-e140d553d425 req-f468fb21-1ebb-4642-a78a-dac962fc30ae service nova] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Received event network-vif-deleted-438d8dc0-9244-438c-b078-fa7754182d12 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 955.075905] env[62499]: INFO nova.compute.manager [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] [instance: 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87] Took 1.02 seconds to deallocate network for instance. [ 955.095742] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Releasing lock "refresh_cache-ca937041-1aa6-46d8-be4e-b4983652fe15" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 955.096173] env[62499]: DEBUG nova.compute.manager [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 955.096370] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 955.096647] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f83ff86b-2a8a-4126-a0d3-2654ff1a322a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 955.105712] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0cceae9-8df2-48b5-96cc-4b010f3ce4d6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 955.128592] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ca937041-1aa6-46d8-be4e-b4983652fe15 could not be found. [ 955.128823] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 955.129010] env[62499]: INFO nova.compute.manager [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Took 0.03 seconds to destroy the instance on the hypervisor. [ 955.129277] env[62499]: DEBUG oslo.service.loopingcall [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 955.129501] env[62499]: DEBUG nova.compute.manager [-] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 955.129595] env[62499]: DEBUG nova.network.neutron [-] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 955.148498] env[62499]: DEBUG nova.network.neutron [-] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 955.167313] env[62499]: DEBUG nova.network.neutron [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 955.257122] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-17ed1b3c-ff09-4dd9-8f3f-23bc567ecdfa {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 955.264463] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0bcae45-00f3-4a3b-a7aa-1c0e397f0e1a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 955.293278] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c206431-ddd0-407d-a22f-54f19c914472 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 955.300034] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ffcbe409-3255-43a8-b657-d9b45f5cae61 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 955.312514] env[62499]: DEBUG nova.compute.provider_tree [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 955.314252] env[62499]: DEBUG nova.network.neutron [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 955.653561] env[62499]: DEBUG nova.network.neutron [-] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 955.816536] env[62499]: DEBUG nova.scheduler.client.report [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 955.819952] env[62499]: DEBUG oslo_concurrency.lockutils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Releasing lock "refresh_cache-9fd7db7e-466e-4213-9e1d-210eae714884" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 955.820223] env[62499]: DEBUG nova.compute.manager [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 955.820366] env[62499]: DEBUG nova.compute.manager [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 955.820529] env[62499]: DEBUG nova.network.neutron [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 955.848632] env[62499]: DEBUG nova.network.neutron [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 956.113146] env[62499]: INFO nova.scheduler.client.report [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Deleted allocations for instance 95a5cb2c-24da-43c0-b9d1-7f0b51ccda87 [ 956.157034] env[62499]: INFO nova.compute.manager [-] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Took 1.03 seconds to deallocate network for instance. [ 956.161752] env[62499]: DEBUG nova.compute.claims [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 956.162167] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 956.324654] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.683s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 956.325347] env[62499]: ERROR nova.compute.manager [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 4464326a-eda6-4130-9df6-a9d552a560c1, please check neutron logs for more information. [ 956.325347] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Traceback (most recent call last): [ 956.325347] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 956.325347] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] self.driver.spawn(context, instance, image_meta, [ 956.325347] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 956.325347] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] self._vmops.spawn(context, instance, image_meta, injected_files, [ 956.325347] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 956.325347] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] vm_ref = self.build_virtual_machine(instance, [ 956.325347] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 956.325347] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] vif_infos = vmwarevif.get_vif_info(self._session, [ 956.325347] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 956.325785] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] for vif in network_info: [ 956.325785] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 956.325785] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] return self._sync_wrapper(fn, *args, **kwargs) [ 956.325785] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 956.325785] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] self.wait() [ 956.325785] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 956.325785] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] self[:] = self._gt.wait() [ 956.325785] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 956.325785] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] return self._exit_event.wait() [ 956.325785] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 956.325785] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] current.throw(*self._exc) [ 956.325785] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 956.325785] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] result = function(*args, **kwargs) [ 956.326154] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 956.326154] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] return func(*args, **kwargs) [ 956.326154] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 956.326154] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] raise e [ 956.326154] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 956.326154] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] nwinfo = self.network_api.allocate_for_instance( [ 956.326154] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 956.326154] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] created_port_ids = self._update_ports_for_instance( [ 956.326154] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 956.326154] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] with excutils.save_and_reraise_exception(): [ 956.326154] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 956.326154] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] self.force_reraise() [ 956.326154] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 956.326504] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] raise self.value [ 956.326504] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 956.326504] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] updated_port = self._update_port( [ 956.326504] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 956.326504] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] _ensure_no_port_binding_failure(port) [ 956.326504] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 956.326504] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] raise exception.PortBindingFailed(port_id=port['id']) [ 956.326504] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] nova.exception.PortBindingFailed: Binding failed for port 4464326a-eda6-4130-9df6-a9d552a560c1, please check neutron logs for more information. [ 956.326504] env[62499]: ERROR nova.compute.manager [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] [ 956.326504] env[62499]: DEBUG nova.compute.utils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Binding failed for port 4464326a-eda6-4130-9df6-a9d552a560c1, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 956.329672] env[62499]: DEBUG nova.compute.manager [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Build of instance ea4b6c62-2880-4a59-8017-b5d1518c1060 was re-scheduled: Binding failed for port 4464326a-eda6-4130-9df6-a9d552a560c1, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 956.330130] env[62499]: DEBUG nova.compute.manager [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 956.330407] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "refresh_cache-ea4b6c62-2880-4a59-8017-b5d1518c1060" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 956.330592] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquired lock "refresh_cache-ea4b6c62-2880-4a59-8017-b5d1518c1060" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 956.330792] env[62499]: DEBUG nova.network.neutron [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 956.331841] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.429s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 956.349967] env[62499]: DEBUG nova.network.neutron [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 956.620348] env[62499]: DEBUG oslo_concurrency.lockutils [None req-499ee740-6bfe-4145-8bdf-393a38067bde tempest-AttachVolumeTestJSON-287116629 tempest-AttachVolumeTestJSON-287116629-project-member] Lock "95a5cb2c-24da-43c0-b9d1-7f0b51ccda87" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 75.831s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 956.853625] env[62499]: INFO nova.compute.manager [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 9fd7db7e-466e-4213-9e1d-210eae714884] Took 1.03 seconds to deallocate network for instance. [ 956.857111] env[62499]: DEBUG nova.network.neutron [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 956.958461] env[62499]: DEBUG nova.network.neutron [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 956.969703] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e438c4c-5e3e-4470-b2cd-31f8b98bd203 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 956.978431] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a421a4f6-6074-4073-8f12-d4b580519521 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 957.008451] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86cbd8cf-3f61-40e4-b4ae-d7e0e7a67c82 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 957.015620] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ede31210-daeb-4da7-81b8-ffd0376301bb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 957.777824] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Releasing lock "refresh_cache-ea4b6c62-2880-4a59-8017-b5d1518c1060" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 957.778114] env[62499]: DEBUG nova.compute.manager [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 957.778277] env[62499]: DEBUG nova.compute.manager [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 957.778445] env[62499]: DEBUG nova.network.neutron [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 957.781352] env[62499]: DEBUG nova.compute.provider_tree [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 957.804863] env[62499]: DEBUG nova.network.neutron [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 958.209406] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 958.209406] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 958.209406] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Starting heal instance info cache {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9928}} [ 958.288601] env[62499]: DEBUG nova.scheduler.client.report [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 958.308328] env[62499]: DEBUG nova.network.neutron [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 958.309915] env[62499]: INFO nova.scheduler.client.report [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Deleted allocations for instance 9fd7db7e-466e-4213-9e1d-210eae714884 [ 958.648243] env[62499]: DEBUG oslo_concurrency.lockutils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Acquiring lock "d9151031-e57a-4bc9-9f6f-ce2d8e74a001" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 958.648515] env[62499]: DEBUG oslo_concurrency.lockutils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Lock "d9151031-e57a-4bc9-9f6f-ce2d8e74a001" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 958.711981] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Didn't find any instances for network info cache update. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10014}} [ 958.712242] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 958.712404] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 958.712551] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 958.712699] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 958.712839] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 958.712981] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 958.713139] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62499) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10547}} [ 958.713613] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager.update_available_resource {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 958.795182] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.462s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 958.795182] env[62499]: ERROR nova.compute.manager [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 461e9d84-836a-4f8b-8ea2-f4ea9f208eef, please check neutron logs for more information. [ 958.795182] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Traceback (most recent call last): [ 958.795182] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 958.795182] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] self.driver.spawn(context, instance, image_meta, [ 958.795182] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 958.795182] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] self._vmops.spawn(context, instance, image_meta, injected_files, [ 958.795182] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 958.795182] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] vm_ref = self.build_virtual_machine(instance, [ 958.795738] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 958.795738] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] vif_infos = vmwarevif.get_vif_info(self._session, [ 958.795738] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 958.795738] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] for vif in network_info: [ 958.795738] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 958.795738] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] return self._sync_wrapper(fn, *args, **kwargs) [ 958.795738] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 958.795738] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] self.wait() [ 958.795738] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 958.795738] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] self[:] = self._gt.wait() [ 958.795738] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 958.795738] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] return self._exit_event.wait() [ 958.795738] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 958.796526] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] current.throw(*self._exc) [ 958.796526] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 958.796526] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] result = function(*args, **kwargs) [ 958.796526] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 958.796526] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] return func(*args, **kwargs) [ 958.796526] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 958.796526] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] raise e [ 958.796526] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 958.796526] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] nwinfo = self.network_api.allocate_for_instance( [ 958.796526] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 958.796526] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] created_port_ids = self._update_ports_for_instance( [ 958.796526] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 958.796526] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] with excutils.save_and_reraise_exception(): [ 958.797515] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 958.797515] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] self.force_reraise() [ 958.797515] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 958.797515] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] raise self.value [ 958.797515] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 958.797515] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] updated_port = self._update_port( [ 958.797515] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 958.797515] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] _ensure_no_port_binding_failure(port) [ 958.797515] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 958.797515] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] raise exception.PortBindingFailed(port_id=port['id']) [ 958.797515] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] nova.exception.PortBindingFailed: Binding failed for port 461e9d84-836a-4f8b-8ea2-f4ea9f208eef, please check neutron logs for more information. [ 958.797515] env[62499]: ERROR nova.compute.manager [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] [ 958.797926] env[62499]: DEBUG nova.compute.utils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Binding failed for port 461e9d84-836a-4f8b-8ea2-f4ea9f208eef, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 958.797926] env[62499]: DEBUG oslo_concurrency.lockutils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 8.251s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 958.799418] env[62499]: DEBUG nova.compute.manager [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Build of instance 99920efd-13e6-4a18-9532-4b53bea3c736 was re-scheduled: Binding failed for port 461e9d84-836a-4f8b-8ea2-f4ea9f208eef, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 958.800123] env[62499]: DEBUG nova.compute.manager [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 958.800123] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquiring lock "refresh_cache-99920efd-13e6-4a18-9532-4b53bea3c736" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 958.800304] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Acquired lock "refresh_cache-99920efd-13e6-4a18-9532-4b53bea3c736" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 958.801276] env[62499]: DEBUG nova.network.neutron [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 958.816640] env[62499]: INFO nova.compute.manager [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: ea4b6c62-2880-4a59-8017-b5d1518c1060] Took 1.04 seconds to deallocate network for instance. [ 958.819623] env[62499]: DEBUG oslo_concurrency.lockutils [None req-53333dec-ce18-43e3-8a0d-bfed00c4be7b tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "9fd7db7e-466e-4213-9e1d-210eae714884" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 44.001s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 959.151133] env[62499]: DEBUG nova.compute.manager [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 959.216624] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 959.321706] env[62499]: DEBUG nova.network.neutron [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 959.405374] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67eb31d0-ac03-4841-b6ba-723b323e4586 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 959.413947] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2b3f2786-d381-4130-9cf6-c7a6320ac8b4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 959.417525] env[62499]: DEBUG nova.network.neutron [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 959.445262] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-51be6078-d63a-4dcb-b71d-b2a408b996d3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 959.453455] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1baf1ca-77c1-4afd-be7b-b31f85e26373 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 959.467224] env[62499]: DEBUG nova.compute.provider_tree [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 959.671453] env[62499]: DEBUG oslo_concurrency.lockutils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 959.853763] env[62499]: INFO nova.scheduler.client.report [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Deleted allocations for instance ea4b6c62-2880-4a59-8017-b5d1518c1060 [ 959.921077] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Releasing lock "refresh_cache-99920efd-13e6-4a18-9532-4b53bea3c736" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 959.921331] env[62499]: DEBUG nova.compute.manager [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 959.921508] env[62499]: DEBUG nova.compute.manager [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 959.921673] env[62499]: DEBUG nova.network.neutron [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 959.936899] env[62499]: DEBUG nova.network.neutron [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 959.970422] env[62499]: DEBUG nova.scheduler.client.report [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 960.363707] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ac9ccd3d-784a-4679-b4a1-84f402696a69 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "ea4b6c62-2880-4a59-8017-b5d1518c1060" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 43.711s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 960.418897] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "1c1379c2-348b-4be8-9415-594ce53ebf55" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 960.419169] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "1c1379c2-348b-4be8-9415-594ce53ebf55" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 960.439670] env[62499]: DEBUG nova.network.neutron [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 960.474721] env[62499]: DEBUG oslo_concurrency.lockutils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.678s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 960.475365] env[62499]: ERROR nova.compute.manager [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 5f7a9791-8f2a-445c-a05f-00d4c5bd13c5, please check neutron logs for more information. [ 960.475365] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Traceback (most recent call last): [ 960.475365] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 960.475365] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] self.driver.spawn(context, instance, image_meta, [ 960.475365] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 960.475365] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 960.475365] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 960.475365] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] vm_ref = self.build_virtual_machine(instance, [ 960.475365] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 960.475365] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] vif_infos = vmwarevif.get_vif_info(self._session, [ 960.475365] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 960.475746] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] for vif in network_info: [ 960.475746] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 960.475746] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] return self._sync_wrapper(fn, *args, **kwargs) [ 960.475746] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 960.475746] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] self.wait() [ 960.475746] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 960.475746] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] self[:] = self._gt.wait() [ 960.475746] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 960.475746] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] return self._exit_event.wait() [ 960.475746] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 960.475746] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] current.throw(*self._exc) [ 960.475746] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 960.475746] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] result = function(*args, **kwargs) [ 960.476146] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 960.476146] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] return func(*args, **kwargs) [ 960.476146] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 960.476146] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] raise e [ 960.476146] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 960.476146] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] nwinfo = self.network_api.allocate_for_instance( [ 960.476146] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 960.476146] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] created_port_ids = self._update_ports_for_instance( [ 960.476146] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 960.476146] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] with excutils.save_and_reraise_exception(): [ 960.476146] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 960.476146] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] self.force_reraise() [ 960.476146] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 960.476578] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] raise self.value [ 960.476578] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 960.476578] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] updated_port = self._update_port( [ 960.476578] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 960.476578] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] _ensure_no_port_binding_failure(port) [ 960.476578] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 960.476578] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] raise exception.PortBindingFailed(port_id=port['id']) [ 960.476578] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] nova.exception.PortBindingFailed: Binding failed for port 5f7a9791-8f2a-445c-a05f-00d4c5bd13c5, please check neutron logs for more information. [ 960.476578] env[62499]: ERROR nova.compute.manager [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] [ 960.476578] env[62499]: DEBUG nova.compute.utils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Binding failed for port 5f7a9791-8f2a-445c-a05f-00d4c5bd13c5, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 960.477158] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 7.348s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 960.478795] env[62499]: INFO nova.compute.claims [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 960.481411] env[62499]: DEBUG nova.compute.manager [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Build of instance 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d was re-scheduled: Binding failed for port 5f7a9791-8f2a-445c-a05f-00d4c5bd13c5, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 960.481815] env[62499]: DEBUG nova.compute.manager [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 960.482053] env[62499]: DEBUG oslo_concurrency.lockutils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Acquiring lock "refresh_cache-4170a2f4-73ff-4bc6-9dc6-d2c4765c449d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 960.482209] env[62499]: DEBUG oslo_concurrency.lockutils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Acquired lock "refresh_cache-4170a2f4-73ff-4bc6-9dc6-d2c4765c449d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 960.482370] env[62499]: DEBUG nova.network.neutron [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 960.922306] env[62499]: DEBUG nova.compute.manager [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 960.942055] env[62499]: INFO nova.compute.manager [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] [instance: 99920efd-13e6-4a18-9532-4b53bea3c736] Took 1.02 seconds to deallocate network for instance. [ 961.001349] env[62499]: DEBUG nova.network.neutron [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 961.084417] env[62499]: DEBUG nova.network.neutron [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 961.443305] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 961.567929] env[62499]: DEBUG oslo_concurrency.lockutils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "7d91c659-d0ac-435c-af73-d6629bcde6bf" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 961.568327] env[62499]: DEBUG oslo_concurrency.lockutils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "7d91c659-d0ac-435c-af73-d6629bcde6bf" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 961.586650] env[62499]: DEBUG oslo_concurrency.lockutils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Releasing lock "refresh_cache-4170a2f4-73ff-4bc6-9dc6-d2c4765c449d" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 961.586863] env[62499]: DEBUG nova.compute.manager [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 961.587056] env[62499]: DEBUG nova.compute.manager [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 961.587228] env[62499]: DEBUG nova.network.neutron [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 961.590712] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d6c6ce8-aaa7-4f62-a481-e989fd35f2ee {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 961.598572] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2f6eae8-c927-4451-a94c-a15425304635 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 961.629841] env[62499]: DEBUG nova.network.neutron [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 961.631452] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-46eb370f-19d6-4476-85d7-41bb1da6418c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 961.639143] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84f5ab86-7a4b-443a-b01f-da386d050d73 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 961.652441] env[62499]: DEBUG nova.compute.provider_tree [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 961.972378] env[62499]: INFO nova.scheduler.client.report [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Deleted allocations for instance 99920efd-13e6-4a18-9532-4b53bea3c736 [ 962.070802] env[62499]: DEBUG nova.compute.manager [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 962.134488] env[62499]: DEBUG nova.network.neutron [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 962.155471] env[62499]: DEBUG nova.scheduler.client.report [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 962.480456] env[62499]: DEBUG oslo_concurrency.lockutils [None req-ad9e2027-8ae3-4bdb-88a8-fe1e94427de8 tempest-AttachVolumeNegativeTest-31807712 tempest-AttachVolumeNegativeTest-31807712-project-member] Lock "99920efd-13e6-4a18-9532-4b53bea3c736" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 42.151s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 962.589506] env[62499]: DEBUG oslo_concurrency.lockutils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 962.636666] env[62499]: INFO nova.compute.manager [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] [instance: 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d] Took 1.05 seconds to deallocate network for instance. [ 962.660634] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.183s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 962.661301] env[62499]: DEBUG nova.compute.manager [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 962.663707] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 9.528s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 962.665623] env[62499]: INFO nova.compute.claims [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 963.169463] env[62499]: DEBUG nova.compute.utils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 963.172523] env[62499]: DEBUG nova.compute.manager [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 963.172691] env[62499]: DEBUG nova.network.neutron [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 963.213079] env[62499]: DEBUG nova.policy [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '10ba90e97f614131adf79a9a8480d702', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '1998d4f886c848e1a219e14fa00ef209', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 963.475017] env[62499]: DEBUG nova.network.neutron [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Successfully created port: 92271e85-7991-4870-b66b-67bbb7cfdd5c {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 963.673498] env[62499]: INFO nova.scheduler.client.report [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Deleted allocations for instance 4170a2f4-73ff-4bc6-9dc6-d2c4765c449d [ 963.684734] env[62499]: DEBUG nova.compute.manager [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 963.783168] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e3e43f9-7a4f-4ece-ab23-a22e71ca015c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 963.789760] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e40a12d-7e54-48d5-9032-b768c1e07e58 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 963.827933] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2c1d01fc-db84-43a1-acec-15b1b57ad831 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 963.836235] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a22ed569-d3db-4a67-b492-97bc17f4587b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 963.849917] env[62499]: DEBUG nova.compute.provider_tree [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 964.152246] env[62499]: DEBUG nova.compute.manager [req-1ef6a956-a56c-4cac-9eb3-12fac3b91c36 req-ef43d923-da2d-4135-ae50-ddfa6101c7ea service nova] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Received event network-changed-92271e85-7991-4870-b66b-67bbb7cfdd5c {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 964.152599] env[62499]: DEBUG nova.compute.manager [req-1ef6a956-a56c-4cac-9eb3-12fac3b91c36 req-ef43d923-da2d-4135-ae50-ddfa6101c7ea service nova] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Refreshing instance network info cache due to event network-changed-92271e85-7991-4870-b66b-67bbb7cfdd5c. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 964.152884] env[62499]: DEBUG oslo_concurrency.lockutils [req-1ef6a956-a56c-4cac-9eb3-12fac3b91c36 req-ef43d923-da2d-4135-ae50-ddfa6101c7ea service nova] Acquiring lock "refresh_cache-636d27e5-f184-4656-9225-09c6ec0f0772" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 964.153049] env[62499]: DEBUG oslo_concurrency.lockutils [req-1ef6a956-a56c-4cac-9eb3-12fac3b91c36 req-ef43d923-da2d-4135-ae50-ddfa6101c7ea service nova] Acquired lock "refresh_cache-636d27e5-f184-4656-9225-09c6ec0f0772" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 964.153246] env[62499]: DEBUG nova.network.neutron [req-1ef6a956-a56c-4cac-9eb3-12fac3b91c36 req-ef43d923-da2d-4135-ae50-ddfa6101c7ea service nova] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Refreshing network info cache for port 92271e85-7991-4870-b66b-67bbb7cfdd5c {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 964.196016] env[62499]: DEBUG oslo_concurrency.lockutils [None req-8aea0860-3bf9-4a17-bdf9-51230c24f2b8 tempest-ServersNegativeTestJSON-1007551573 tempest-ServersNegativeTestJSON-1007551573-project-member] Lock "4170a2f4-73ff-4bc6-9dc6-d2c4765c449d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 34.733s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 964.335480] env[62499]: ERROR nova.compute.manager [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 92271e85-7991-4870-b66b-67bbb7cfdd5c, please check neutron logs for more information. [ 964.335480] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 964.335480] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 964.335480] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 964.335480] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 964.335480] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 964.335480] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 964.335480] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 964.335480] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 964.335480] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 964.335480] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 964.335480] env[62499]: ERROR nova.compute.manager raise self.value [ 964.335480] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 964.335480] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 964.335480] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 964.335480] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 964.335994] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 964.335994] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 964.335994] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 92271e85-7991-4870-b66b-67bbb7cfdd5c, please check neutron logs for more information. [ 964.335994] env[62499]: ERROR nova.compute.manager [ 964.335994] env[62499]: Traceback (most recent call last): [ 964.335994] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 964.335994] env[62499]: listener.cb(fileno) [ 964.335994] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 964.335994] env[62499]: result = function(*args, **kwargs) [ 964.335994] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 964.335994] env[62499]: return func(*args, **kwargs) [ 964.335994] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 964.335994] env[62499]: raise e [ 964.335994] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 964.335994] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 964.335994] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 964.335994] env[62499]: created_port_ids = self._update_ports_for_instance( [ 964.335994] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 964.335994] env[62499]: with excutils.save_and_reraise_exception(): [ 964.335994] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 964.335994] env[62499]: self.force_reraise() [ 964.335994] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 964.335994] env[62499]: raise self.value [ 964.335994] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 964.335994] env[62499]: updated_port = self._update_port( [ 964.335994] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 964.335994] env[62499]: _ensure_no_port_binding_failure(port) [ 964.335994] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 964.335994] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 964.336876] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 92271e85-7991-4870-b66b-67bbb7cfdd5c, please check neutron logs for more information. [ 964.336876] env[62499]: Removing descriptor: 17 [ 964.353479] env[62499]: DEBUG nova.scheduler.client.report [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 964.675310] env[62499]: DEBUG nova.network.neutron [req-1ef6a956-a56c-4cac-9eb3-12fac3b91c36 req-ef43d923-da2d-4135-ae50-ddfa6101c7ea service nova] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 964.694342] env[62499]: DEBUG nova.compute.manager [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 964.719112] env[62499]: DEBUG nova.virt.hardware [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 964.719112] env[62499]: DEBUG nova.virt.hardware [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 964.719112] env[62499]: DEBUG nova.virt.hardware [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 964.719270] env[62499]: DEBUG nova.virt.hardware [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 964.719270] env[62499]: DEBUG nova.virt.hardware [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 964.719270] env[62499]: DEBUG nova.virt.hardware [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 964.719786] env[62499]: DEBUG nova.virt.hardware [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 964.720196] env[62499]: DEBUG nova.virt.hardware [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 964.723017] env[62499]: DEBUG nova.virt.hardware [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 964.723017] env[62499]: DEBUG nova.virt.hardware [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 964.723017] env[62499]: DEBUG nova.virt.hardware [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 964.723017] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dfc469f9-5d83-41ff-9c7b-3102ce189173 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 964.730459] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a3e9872-e93c-454d-8491-caf01da6092a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 964.746646] env[62499]: ERROR nova.compute.manager [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 92271e85-7991-4870-b66b-67bbb7cfdd5c, please check neutron logs for more information. [ 964.746646] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Traceback (most recent call last): [ 964.746646] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 964.746646] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] yield resources [ 964.746646] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 964.746646] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] self.driver.spawn(context, instance, image_meta, [ 964.746646] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 964.746646] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] self._vmops.spawn(context, instance, image_meta, injected_files, [ 964.746646] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 964.746646] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] vm_ref = self.build_virtual_machine(instance, [ 964.746646] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 964.747274] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] vif_infos = vmwarevif.get_vif_info(self._session, [ 964.747274] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 964.747274] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] for vif in network_info: [ 964.747274] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 964.747274] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] return self._sync_wrapper(fn, *args, **kwargs) [ 964.747274] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 964.747274] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] self.wait() [ 964.747274] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 964.747274] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] self[:] = self._gt.wait() [ 964.747274] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 964.747274] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] return self._exit_event.wait() [ 964.747274] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 964.747274] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] current.throw(*self._exc) [ 964.749443] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 964.749443] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] result = function(*args, **kwargs) [ 964.749443] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 964.749443] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] return func(*args, **kwargs) [ 964.749443] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 964.749443] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] raise e [ 964.749443] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 964.749443] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] nwinfo = self.network_api.allocate_for_instance( [ 964.749443] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 964.749443] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] created_port_ids = self._update_ports_for_instance( [ 964.749443] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 964.749443] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] with excutils.save_and_reraise_exception(): [ 964.749443] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 964.749819] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] self.force_reraise() [ 964.749819] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 964.749819] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] raise self.value [ 964.749819] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 964.749819] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] updated_port = self._update_port( [ 964.749819] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 964.749819] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] _ensure_no_port_binding_failure(port) [ 964.749819] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 964.749819] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] raise exception.PortBindingFailed(port_id=port['id']) [ 964.749819] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] nova.exception.PortBindingFailed: Binding failed for port 92271e85-7991-4870-b66b-67bbb7cfdd5c, please check neutron logs for more information. [ 964.749819] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] [ 964.749819] env[62499]: INFO nova.compute.manager [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Terminating instance [ 964.750890] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Acquiring lock "refresh_cache-636d27e5-f184-4656-9225-09c6ec0f0772" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 964.960845] env[62499]: DEBUG nova.network.neutron [req-1ef6a956-a56c-4cac-9eb3-12fac3b91c36 req-ef43d923-da2d-4135-ae50-ddfa6101c7ea service nova] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 964.960845] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.195s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 964.960845] env[62499]: DEBUG nova.compute.manager [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 964.960845] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 8.706s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 965.297244] env[62499]: DEBUG oslo_concurrency.lockutils [req-1ef6a956-a56c-4cac-9eb3-12fac3b91c36 req-ef43d923-da2d-4135-ae50-ddfa6101c7ea service nova] Releasing lock "refresh_cache-636d27e5-f184-4656-9225-09c6ec0f0772" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 965.297703] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Acquired lock "refresh_cache-636d27e5-f184-4656-9225-09c6ec0f0772" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 965.297902] env[62499]: DEBUG nova.network.neutron [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 965.377144] env[62499]: DEBUG nova.compute.utils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 965.382399] env[62499]: DEBUG nova.compute.manager [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 965.382595] env[62499]: DEBUG nova.network.neutron [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 965.435742] env[62499]: DEBUG nova.policy [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4762545dc5e14c16b5cbb1bad27d90a2', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3f11318c300c4c308f3f8a14db355954', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 965.476817] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b64f6522-243b-476f-bea7-11b79fd03d30 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 965.485997] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-607588f7-d0d7-4f7b-b5f3-ff5301efda44 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 965.516616] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-df1b1b7b-bdb7-4127-a294-23f0444706d9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 965.524370] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-326c240a-8446-451c-b710-eed05429443a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 965.540391] env[62499]: DEBUG nova.compute.provider_tree [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 965.693286] env[62499]: DEBUG nova.network.neutron [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Successfully created port: b92e7d6b-5c1d-4570-ab7e-15c081e63812 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 965.821915] env[62499]: DEBUG nova.network.neutron [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 965.889440] env[62499]: DEBUG nova.compute.manager [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 965.907671] env[62499]: DEBUG nova.network.neutron [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 966.043616] env[62499]: DEBUG nova.scheduler.client.report [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 966.175999] env[62499]: DEBUG nova.compute.manager [req-aa0fbd77-5b28-4f3d-8bf1-3003bacfda93 req-b33a3d2b-695a-4c11-b97e-131900f0a700 service nova] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Received event network-vif-deleted-92271e85-7991-4870-b66b-67bbb7cfdd5c {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 966.411019] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Releasing lock "refresh_cache-636d27e5-f184-4656-9225-09c6ec0f0772" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 966.414107] env[62499]: DEBUG nova.compute.manager [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 966.414107] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 966.414107] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-910a45f2-d8db-4793-bfe6-5ded3d4f8582 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 966.423838] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8973c40f-173e-422a-a0e8-dcdf3fd7e737 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 966.447600] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 636d27e5-f184-4656-9225-09c6ec0f0772 could not be found. [ 966.447725] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 966.447969] env[62499]: INFO nova.compute.manager [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Took 0.04 seconds to destroy the instance on the hypervisor. [ 966.448301] env[62499]: DEBUG oslo.service.loopingcall [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 966.448583] env[62499]: DEBUG nova.compute.manager [-] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 966.448730] env[62499]: DEBUG nova.network.neutron [-] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 966.467316] env[62499]: DEBUG nova.network.neutron [-] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 966.548801] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.680s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 966.549401] env[62499]: ERROR nova.compute.manager [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 438d8dc0-9244-438c-b078-fa7754182d12, please check neutron logs for more information. [ 966.549401] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Traceback (most recent call last): [ 966.549401] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 966.549401] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] self.driver.spawn(context, instance, image_meta, [ 966.549401] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 966.549401] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] self._vmops.spawn(context, instance, image_meta, injected_files, [ 966.549401] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 966.549401] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] vm_ref = self.build_virtual_machine(instance, [ 966.549401] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 966.549401] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] vif_infos = vmwarevif.get_vif_info(self._session, [ 966.549401] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 966.549737] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] for vif in network_info: [ 966.549737] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 966.549737] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] return self._sync_wrapper(fn, *args, **kwargs) [ 966.549737] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 966.549737] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] self.wait() [ 966.549737] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 966.549737] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] self[:] = self._gt.wait() [ 966.549737] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 966.549737] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] return self._exit_event.wait() [ 966.549737] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 966.549737] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] current.throw(*self._exc) [ 966.549737] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 966.549737] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] result = function(*args, **kwargs) [ 966.550179] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 966.550179] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] return func(*args, **kwargs) [ 966.550179] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 966.550179] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] raise e [ 966.550179] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 966.550179] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] nwinfo = self.network_api.allocate_for_instance( [ 966.550179] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 966.550179] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] created_port_ids = self._update_ports_for_instance( [ 966.550179] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 966.550179] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] with excutils.save_and_reraise_exception(): [ 966.550179] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 966.550179] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] self.force_reraise() [ 966.550179] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 966.550585] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] raise self.value [ 966.550585] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 966.550585] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] updated_port = self._update_port( [ 966.550585] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 966.550585] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] _ensure_no_port_binding_failure(port) [ 966.550585] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 966.550585] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] raise exception.PortBindingFailed(port_id=port['id']) [ 966.550585] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] nova.exception.PortBindingFailed: Binding failed for port 438d8dc0-9244-438c-b078-fa7754182d12, please check neutron logs for more information. [ 966.550585] env[62499]: ERROR nova.compute.manager [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] [ 966.550585] env[62499]: DEBUG nova.compute.utils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Binding failed for port 438d8dc0-9244-438c-b078-fa7754182d12, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 966.551844] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 7.335s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 966.552066] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 966.552259] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62499) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 966.552570] env[62499]: DEBUG oslo_concurrency.lockutils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 6.881s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 966.554216] env[62499]: INFO nova.compute.claims [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 966.556969] env[62499]: DEBUG nova.compute.manager [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Build of instance ca937041-1aa6-46d8-be4e-b4983652fe15 was re-scheduled: Binding failed for port 438d8dc0-9244-438c-b078-fa7754182d12, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 966.557477] env[62499]: DEBUG nova.compute.manager [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 966.558121] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "refresh_cache-ca937041-1aa6-46d8-be4e-b4983652fe15" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 966.558121] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquired lock "refresh_cache-ca937041-1aa6-46d8-be4e-b4983652fe15" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 966.558121] env[62499]: DEBUG nova.network.neutron [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 966.559525] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f413e69-6858-49a0-a53e-31132c9288de {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 966.568101] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44a5b2e4-de2a-4111-9278-fb3f62caa446 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 966.582909] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2c58af53-d2f6-46a6-85db-7bdb38b08fa6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 966.589536] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af082e7e-eec1-401c-b011-3875aed5a5df {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 966.624013] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181514MB free_disk=130GB free_vcpus=48 pci_devices=None {{(pid=62499) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 966.624013] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 966.707044] env[62499]: ERROR nova.compute.manager [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b92e7d6b-5c1d-4570-ab7e-15c081e63812, please check neutron logs for more information. [ 966.707044] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 966.707044] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 966.707044] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 966.707044] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 966.707044] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 966.707044] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 966.707044] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 966.707044] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 966.707044] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 966.707044] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 966.707044] env[62499]: ERROR nova.compute.manager raise self.value [ 966.707044] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 966.707044] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 966.707044] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 966.707044] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 966.707771] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 966.707771] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 966.707771] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b92e7d6b-5c1d-4570-ab7e-15c081e63812, please check neutron logs for more information. [ 966.707771] env[62499]: ERROR nova.compute.manager [ 966.707771] env[62499]: Traceback (most recent call last): [ 966.707771] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 966.707771] env[62499]: listener.cb(fileno) [ 966.707771] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 966.707771] env[62499]: result = function(*args, **kwargs) [ 966.707771] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 966.707771] env[62499]: return func(*args, **kwargs) [ 966.707771] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 966.707771] env[62499]: raise e [ 966.707771] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 966.707771] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 966.707771] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 966.707771] env[62499]: created_port_ids = self._update_ports_for_instance( [ 966.707771] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 966.707771] env[62499]: with excutils.save_and_reraise_exception(): [ 966.707771] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 966.707771] env[62499]: self.force_reraise() [ 966.707771] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 966.707771] env[62499]: raise self.value [ 966.707771] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 966.707771] env[62499]: updated_port = self._update_port( [ 966.707771] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 966.707771] env[62499]: _ensure_no_port_binding_failure(port) [ 966.707771] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 966.707771] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 966.709014] env[62499]: nova.exception.PortBindingFailed: Binding failed for port b92e7d6b-5c1d-4570-ab7e-15c081e63812, please check neutron logs for more information. [ 966.709014] env[62499]: Removing descriptor: 17 [ 966.897093] env[62499]: DEBUG nova.compute.manager [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 966.932384] env[62499]: DEBUG nova.virt.hardware [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 966.932635] env[62499]: DEBUG nova.virt.hardware [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 966.932793] env[62499]: DEBUG nova.virt.hardware [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 966.932971] env[62499]: DEBUG nova.virt.hardware [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 966.933178] env[62499]: DEBUG nova.virt.hardware [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 966.933342] env[62499]: DEBUG nova.virt.hardware [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 966.933546] env[62499]: DEBUG nova.virt.hardware [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 966.933703] env[62499]: DEBUG nova.virt.hardware [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 966.933868] env[62499]: DEBUG nova.virt.hardware [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 966.934041] env[62499]: DEBUG nova.virt.hardware [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 966.934229] env[62499]: DEBUG nova.virt.hardware [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 966.935078] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a081a5e6-efe2-4671-8c64-17834f149a2d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 966.943197] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bbba2ae0-cab4-4156-bb3a-14ce981cd5d5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 966.957542] env[62499]: ERROR nova.compute.manager [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b92e7d6b-5c1d-4570-ab7e-15c081e63812, please check neutron logs for more information. [ 966.957542] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Traceback (most recent call last): [ 966.957542] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 966.957542] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] yield resources [ 966.957542] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 966.957542] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] self.driver.spawn(context, instance, image_meta, [ 966.957542] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 966.957542] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] self._vmops.spawn(context, instance, image_meta, injected_files, [ 966.957542] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 966.957542] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] vm_ref = self.build_virtual_machine(instance, [ 966.957542] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 966.958237] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] vif_infos = vmwarevif.get_vif_info(self._session, [ 966.958237] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 966.958237] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] for vif in network_info: [ 966.958237] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 966.958237] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] return self._sync_wrapper(fn, *args, **kwargs) [ 966.958237] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 966.958237] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] self.wait() [ 966.958237] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 966.958237] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] self[:] = self._gt.wait() [ 966.958237] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 966.958237] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] return self._exit_event.wait() [ 966.958237] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 966.958237] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] current.throw(*self._exc) [ 966.958779] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 966.958779] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] result = function(*args, **kwargs) [ 966.958779] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 966.958779] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] return func(*args, **kwargs) [ 966.958779] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 966.958779] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] raise e [ 966.958779] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 966.958779] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] nwinfo = self.network_api.allocate_for_instance( [ 966.958779] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 966.958779] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] created_port_ids = self._update_ports_for_instance( [ 966.958779] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 966.958779] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] with excutils.save_and_reraise_exception(): [ 966.958779] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 966.959176] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] self.force_reraise() [ 966.959176] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 966.959176] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] raise self.value [ 966.959176] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 966.959176] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] updated_port = self._update_port( [ 966.959176] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 966.959176] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] _ensure_no_port_binding_failure(port) [ 966.959176] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 966.959176] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] raise exception.PortBindingFailed(port_id=port['id']) [ 966.959176] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] nova.exception.PortBindingFailed: Binding failed for port b92e7d6b-5c1d-4570-ab7e-15c081e63812, please check neutron logs for more information. [ 966.959176] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] [ 966.959176] env[62499]: INFO nova.compute.manager [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Terminating instance [ 966.959984] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "refresh_cache-b4e185e6-b177-4212-bad8-b9fd1b51ea95" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 966.960282] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquired lock "refresh_cache-b4e185e6-b177-4212-bad8-b9fd1b51ea95" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 966.960532] env[62499]: DEBUG nova.network.neutron [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 966.969132] env[62499]: DEBUG nova.network.neutron [-] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 967.081682] env[62499]: DEBUG nova.network.neutron [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 967.194355] env[62499]: DEBUG nova.network.neutron [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 967.472056] env[62499]: INFO nova.compute.manager [-] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Took 1.02 seconds to deallocate network for instance. [ 967.474245] env[62499]: DEBUG nova.compute.claims [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 967.474430] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 967.477029] env[62499]: DEBUG nova.network.neutron [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 967.546268] env[62499]: DEBUG nova.network.neutron [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 967.685843] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75ea8017-480c-4ef9-9dbb-a8f75c1e9ec4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.693423] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72bd2148-4761-4971-8048-c54dcd7e0c41 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.696579] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Releasing lock "refresh_cache-ca937041-1aa6-46d8-be4e-b4983652fe15" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 967.696791] env[62499]: DEBUG nova.compute.manager [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 967.696967] env[62499]: DEBUG nova.compute.manager [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 967.697145] env[62499]: DEBUG nova.network.neutron [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 967.726012] env[62499]: DEBUG nova.network.neutron [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 967.727573] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f518f9ef-2f11-4dc6-8fb6-3675660323f2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.734906] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-73fd5f03-3be6-4c78-9870-b279668af345 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.748518] env[62499]: DEBUG nova.compute.provider_tree [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 968.049060] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Releasing lock "refresh_cache-b4e185e6-b177-4212-bad8-b9fd1b51ea95" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 968.049060] env[62499]: DEBUG nova.compute.manager [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 968.049060] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 968.049369] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f6744cef-74e3-405f-8775-b40e19e5a8f0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 968.058634] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d8841da-d86a-4808-a820-dc71d1a68b8d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 968.078583] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b4e185e6-b177-4212-bad8-b9fd1b51ea95 could not be found. [ 968.078782] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 968.078961] env[62499]: INFO nova.compute.manager [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Took 0.03 seconds to destroy the instance on the hypervisor. [ 968.079209] env[62499]: DEBUG oslo.service.loopingcall [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 968.079419] env[62499]: DEBUG nova.compute.manager [-] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 968.079510] env[62499]: DEBUG nova.network.neutron [-] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 968.227534] env[62499]: DEBUG nova.network.neutron [-] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 968.230700] env[62499]: DEBUG nova.network.neutron [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 968.247108] env[62499]: DEBUG nova.compute.manager [req-83aed79c-0bcc-4204-ae14-2f563e0a8adf req-c08331f2-a27c-4c5b-b9d5-b4cb09c9739e service nova] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Received event network-changed-b92e7d6b-5c1d-4570-ab7e-15c081e63812 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 968.247337] env[62499]: DEBUG nova.compute.manager [req-83aed79c-0bcc-4204-ae14-2f563e0a8adf req-c08331f2-a27c-4c5b-b9d5-b4cb09c9739e service nova] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Refreshing instance network info cache due to event network-changed-b92e7d6b-5c1d-4570-ab7e-15c081e63812. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 968.247521] env[62499]: DEBUG oslo_concurrency.lockutils [req-83aed79c-0bcc-4204-ae14-2f563e0a8adf req-c08331f2-a27c-4c5b-b9d5-b4cb09c9739e service nova] Acquiring lock "refresh_cache-b4e185e6-b177-4212-bad8-b9fd1b51ea95" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 968.247682] env[62499]: DEBUG oslo_concurrency.lockutils [req-83aed79c-0bcc-4204-ae14-2f563e0a8adf req-c08331f2-a27c-4c5b-b9d5-b4cb09c9739e service nova] Acquired lock "refresh_cache-b4e185e6-b177-4212-bad8-b9fd1b51ea95" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 968.247810] env[62499]: DEBUG nova.network.neutron [req-83aed79c-0bcc-4204-ae14-2f563e0a8adf req-c08331f2-a27c-4c5b-b9d5-b4cb09c9739e service nova] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Refreshing network info cache for port b92e7d6b-5c1d-4570-ab7e-15c081e63812 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 968.251136] env[62499]: DEBUG nova.scheduler.client.report [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 968.731024] env[62499]: DEBUG nova.network.neutron [-] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 968.733096] env[62499]: INFO nova.compute.manager [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: ca937041-1aa6-46d8-be4e-b4983652fe15] Took 1.04 seconds to deallocate network for instance. [ 968.756042] env[62499]: DEBUG oslo_concurrency.lockutils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.203s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 968.756042] env[62499]: DEBUG nova.compute.manager [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 968.758474] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 7.315s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 968.760316] env[62499]: INFO nova.compute.claims [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 968.763896] env[62499]: DEBUG nova.network.neutron [req-83aed79c-0bcc-4204-ae14-2f563e0a8adf req-c08331f2-a27c-4c5b-b9d5-b4cb09c9739e service nova] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 968.833163] env[62499]: DEBUG nova.network.neutron [req-83aed79c-0bcc-4204-ae14-2f563e0a8adf req-c08331f2-a27c-4c5b-b9d5-b4cb09c9739e service nova] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 969.232562] env[62499]: INFO nova.compute.manager [-] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Took 1.15 seconds to deallocate network for instance. [ 969.234908] env[62499]: DEBUG nova.compute.claims [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 969.235104] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 969.264555] env[62499]: DEBUG nova.compute.utils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 969.268044] env[62499]: DEBUG nova.compute.manager [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 969.268166] env[62499]: DEBUG nova.network.neutron [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 969.306745] env[62499]: DEBUG nova.policy [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '48b2a1fbea0e4e22bfa32ec31e438f6d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c21dbc475b0d4225b920fbad495b785a', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 969.335076] env[62499]: DEBUG oslo_concurrency.lockutils [req-83aed79c-0bcc-4204-ae14-2f563e0a8adf req-c08331f2-a27c-4c5b-b9d5-b4cb09c9739e service nova] Releasing lock "refresh_cache-b4e185e6-b177-4212-bad8-b9fd1b51ea95" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 969.335314] env[62499]: DEBUG nova.compute.manager [req-83aed79c-0bcc-4204-ae14-2f563e0a8adf req-c08331f2-a27c-4c5b-b9d5-b4cb09c9739e service nova] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Received event network-vif-deleted-b92e7d6b-5c1d-4570-ab7e-15c081e63812 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 969.641398] env[62499]: DEBUG nova.network.neutron [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Successfully created port: 84cefa2b-4e20-45df-a4c6-1b623a06142c {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 969.768646] env[62499]: DEBUG nova.compute.manager [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 969.777284] env[62499]: INFO nova.scheduler.client.report [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Deleted allocations for instance ca937041-1aa6-46d8-be4e-b4983652fe15 [ 969.864585] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-000028fe-4773-45fe-be94-d91ca2d4e3d1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 969.873108] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91b06570-9106-4153-95db-49563a3c6142 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 969.912409] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e3efdce-87db-4bfb-8ba4-8fe9e103e2e4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 969.919787] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21b6ed59-792f-459d-9745-265f7b0d44d8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 969.934904] env[62499]: DEBUG nova.compute.provider_tree [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 970.290484] env[62499]: DEBUG oslo_concurrency.lockutils [None req-1e3aa116-f163-4221-b591-3b122d27a0a6 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "ca937041-1aa6-46d8-be4e-b4983652fe15" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 38.802s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 970.361463] env[62499]: DEBUG nova.compute.manager [req-e20511ce-6b9a-489f-96a2-96fe03ea88ba req-b0445bd4-b8e3-458b-930c-0776f45083f1 service nova] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Received event network-changed-84cefa2b-4e20-45df-a4c6-1b623a06142c {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 970.361968] env[62499]: DEBUG nova.compute.manager [req-e20511ce-6b9a-489f-96a2-96fe03ea88ba req-b0445bd4-b8e3-458b-930c-0776f45083f1 service nova] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Refreshing instance network info cache due to event network-changed-84cefa2b-4e20-45df-a4c6-1b623a06142c. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 970.361968] env[62499]: DEBUG oslo_concurrency.lockutils [req-e20511ce-6b9a-489f-96a2-96fe03ea88ba req-b0445bd4-b8e3-458b-930c-0776f45083f1 service nova] Acquiring lock "refresh_cache-d9151031-e57a-4bc9-9f6f-ce2d8e74a001" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 970.362160] env[62499]: DEBUG oslo_concurrency.lockutils [req-e20511ce-6b9a-489f-96a2-96fe03ea88ba req-b0445bd4-b8e3-458b-930c-0776f45083f1 service nova] Acquired lock "refresh_cache-d9151031-e57a-4bc9-9f6f-ce2d8e74a001" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 970.362219] env[62499]: DEBUG nova.network.neutron [req-e20511ce-6b9a-489f-96a2-96fe03ea88ba req-b0445bd4-b8e3-458b-930c-0776f45083f1 service nova] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Refreshing network info cache for port 84cefa2b-4e20-45df-a4c6-1b623a06142c {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 970.438072] env[62499]: DEBUG nova.scheduler.client.report [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 970.581289] env[62499]: ERROR nova.compute.manager [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 84cefa2b-4e20-45df-a4c6-1b623a06142c, please check neutron logs for more information. [ 970.581289] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 970.581289] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 970.581289] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 970.581289] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 970.581289] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 970.581289] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 970.581289] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 970.581289] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 970.581289] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 970.581289] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 970.581289] env[62499]: ERROR nova.compute.manager raise self.value [ 970.581289] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 970.581289] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 970.581289] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 970.581289] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 970.581975] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 970.581975] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 970.581975] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 84cefa2b-4e20-45df-a4c6-1b623a06142c, please check neutron logs for more information. [ 970.581975] env[62499]: ERROR nova.compute.manager [ 970.581975] env[62499]: Traceback (most recent call last): [ 970.581975] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 970.581975] env[62499]: listener.cb(fileno) [ 970.581975] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 970.581975] env[62499]: result = function(*args, **kwargs) [ 970.581975] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 970.581975] env[62499]: return func(*args, **kwargs) [ 970.581975] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 970.581975] env[62499]: raise e [ 970.581975] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 970.581975] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 970.581975] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 970.581975] env[62499]: created_port_ids = self._update_ports_for_instance( [ 970.581975] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 970.581975] env[62499]: with excutils.save_and_reraise_exception(): [ 970.581975] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 970.581975] env[62499]: self.force_reraise() [ 970.581975] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 970.581975] env[62499]: raise self.value [ 970.581975] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 970.581975] env[62499]: updated_port = self._update_port( [ 970.581975] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 970.581975] env[62499]: _ensure_no_port_binding_failure(port) [ 970.581975] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 970.581975] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 970.582981] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 84cefa2b-4e20-45df-a4c6-1b623a06142c, please check neutron logs for more information. [ 970.582981] env[62499]: Removing descriptor: 17 [ 970.781251] env[62499]: DEBUG nova.compute.manager [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 970.801331] env[62499]: DEBUG nova.virt.hardware [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 970.801331] env[62499]: DEBUG nova.virt.hardware [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 970.801527] env[62499]: DEBUG nova.virt.hardware [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 970.801574] env[62499]: DEBUG nova.virt.hardware [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 970.802147] env[62499]: DEBUG nova.virt.hardware [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 970.802147] env[62499]: DEBUG nova.virt.hardware [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 970.802147] env[62499]: DEBUG nova.virt.hardware [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 970.802327] env[62499]: DEBUG nova.virt.hardware [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 970.802367] env[62499]: DEBUG nova.virt.hardware [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 970.802509] env[62499]: DEBUG nova.virt.hardware [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 970.802678] env[62499]: DEBUG nova.virt.hardware [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 970.803658] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1401b6f5-2bac-4679-ac89-07315fbcafd6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 970.811425] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f065748e-2fe6-434f-a37a-1fa804c07d47 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 970.824610] env[62499]: ERROR nova.compute.manager [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 84cefa2b-4e20-45df-a4c6-1b623a06142c, please check neutron logs for more information. [ 970.824610] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Traceback (most recent call last): [ 970.824610] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 970.824610] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] yield resources [ 970.824610] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 970.824610] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] self.driver.spawn(context, instance, image_meta, [ 970.824610] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 970.824610] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] self._vmops.spawn(context, instance, image_meta, injected_files, [ 970.824610] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 970.824610] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] vm_ref = self.build_virtual_machine(instance, [ 970.824610] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 970.825076] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] vif_infos = vmwarevif.get_vif_info(self._session, [ 970.825076] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 970.825076] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] for vif in network_info: [ 970.825076] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 970.825076] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] return self._sync_wrapper(fn, *args, **kwargs) [ 970.825076] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 970.825076] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] self.wait() [ 970.825076] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 970.825076] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] self[:] = self._gt.wait() [ 970.825076] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 970.825076] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] return self._exit_event.wait() [ 970.825076] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 970.825076] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] current.throw(*self._exc) [ 970.825707] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 970.825707] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] result = function(*args, **kwargs) [ 970.825707] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 970.825707] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] return func(*args, **kwargs) [ 970.825707] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 970.825707] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] raise e [ 970.825707] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 970.825707] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] nwinfo = self.network_api.allocate_for_instance( [ 970.825707] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 970.825707] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] created_port_ids = self._update_ports_for_instance( [ 970.825707] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 970.825707] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] with excutils.save_and_reraise_exception(): [ 970.825707] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 970.826219] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] self.force_reraise() [ 970.826219] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 970.826219] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] raise self.value [ 970.826219] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 970.826219] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] updated_port = self._update_port( [ 970.826219] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 970.826219] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] _ensure_no_port_binding_failure(port) [ 970.826219] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 970.826219] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] raise exception.PortBindingFailed(port_id=port['id']) [ 970.826219] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] nova.exception.PortBindingFailed: Binding failed for port 84cefa2b-4e20-45df-a4c6-1b623a06142c, please check neutron logs for more information. [ 970.826219] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] [ 970.826219] env[62499]: INFO nova.compute.manager [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Terminating instance [ 970.826769] env[62499]: DEBUG oslo_concurrency.lockutils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Acquiring lock "refresh_cache-d9151031-e57a-4bc9-9f6f-ce2d8e74a001" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 970.878526] env[62499]: DEBUG nova.network.neutron [req-e20511ce-6b9a-489f-96a2-96fe03ea88ba req-b0445bd4-b8e3-458b-930c-0776f45083f1 service nova] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 970.943392] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.185s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 970.943965] env[62499]: DEBUG nova.compute.manager [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 970.946582] env[62499]: DEBUG oslo_concurrency.lockutils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 8.357s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 970.948051] env[62499]: INFO nova.compute.claims [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 970.964106] env[62499]: DEBUG nova.network.neutron [req-e20511ce-6b9a-489f-96a2-96fe03ea88ba req-b0445bd4-b8e3-458b-930c-0776f45083f1 service nova] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 971.452484] env[62499]: DEBUG nova.compute.utils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 971.457707] env[62499]: DEBUG nova.compute.manager [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 971.457707] env[62499]: DEBUG nova.network.neutron [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 971.466654] env[62499]: DEBUG oslo_concurrency.lockutils [req-e20511ce-6b9a-489f-96a2-96fe03ea88ba req-b0445bd4-b8e3-458b-930c-0776f45083f1 service nova] Releasing lock "refresh_cache-d9151031-e57a-4bc9-9f6f-ce2d8e74a001" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 971.467279] env[62499]: DEBUG oslo_concurrency.lockutils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Acquired lock "refresh_cache-d9151031-e57a-4bc9-9f6f-ce2d8e74a001" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 971.467885] env[62499]: DEBUG nova.network.neutron [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 971.495939] env[62499]: DEBUG nova.policy [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c92b9d37f796479ca3b6d9f0ed1789c8', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e444950a637b4fdab2edb1d62aac4be5', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 971.753618] env[62499]: DEBUG nova.network.neutron [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Successfully created port: 9d32fadd-a63a-4b82-a49c-297505381d88 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 971.956737] env[62499]: DEBUG nova.compute.manager [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 972.006376] env[62499]: DEBUG nova.network.neutron [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 972.076805] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-938d6ffc-0275-49ce-8304-413febf71316 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.085708] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 972.085835] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 972.090588] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52a63cd6-d620-46e1-bc23-0bf4e662f857 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.122610] env[62499]: DEBUG nova.network.neutron [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 972.124722] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7fd1b2dd-37da-4ceb-8bf5-6e330f5199d9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.133830] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69b951cf-9533-4082-9670-4f6f6c8f9cdf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.148705] env[62499]: DEBUG nova.compute.provider_tree [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 972.387714] env[62499]: DEBUG nova.compute.manager [req-4eabd9bb-07fe-4688-ab78-c566857a0dbb req-24a76fd4-1059-4306-8e02-c0b47bc3ff00 service nova] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Received event network-vif-deleted-84cefa2b-4e20-45df-a4c6-1b623a06142c {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 972.588757] env[62499]: DEBUG nova.compute.manager [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 972.603251] env[62499]: ERROR nova.compute.manager [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 9d32fadd-a63a-4b82-a49c-297505381d88, please check neutron logs for more information. [ 972.603251] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 972.603251] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 972.603251] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 972.603251] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 972.603251] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 972.603251] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 972.603251] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 972.603251] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 972.603251] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 972.603251] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 972.603251] env[62499]: ERROR nova.compute.manager raise self.value [ 972.603251] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 972.603251] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 972.603251] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 972.603251] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 972.604131] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 972.604131] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 972.604131] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 9d32fadd-a63a-4b82-a49c-297505381d88, please check neutron logs for more information. [ 972.604131] env[62499]: ERROR nova.compute.manager [ 972.604131] env[62499]: Traceback (most recent call last): [ 972.604131] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 972.604131] env[62499]: listener.cb(fileno) [ 972.604131] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 972.604131] env[62499]: result = function(*args, **kwargs) [ 972.604131] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 972.604131] env[62499]: return func(*args, **kwargs) [ 972.604131] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 972.604131] env[62499]: raise e [ 972.604131] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 972.604131] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 972.604131] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 972.604131] env[62499]: created_port_ids = self._update_ports_for_instance( [ 972.604131] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 972.604131] env[62499]: with excutils.save_and_reraise_exception(): [ 972.604131] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 972.604131] env[62499]: self.force_reraise() [ 972.604131] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 972.604131] env[62499]: raise self.value [ 972.604131] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 972.604131] env[62499]: updated_port = self._update_port( [ 972.604131] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 972.604131] env[62499]: _ensure_no_port_binding_failure(port) [ 972.604131] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 972.604131] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 972.604993] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 9d32fadd-a63a-4b82-a49c-297505381d88, please check neutron logs for more information. [ 972.604993] env[62499]: Removing descriptor: 17 [ 972.628042] env[62499]: DEBUG oslo_concurrency.lockutils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Releasing lock "refresh_cache-d9151031-e57a-4bc9-9f6f-ce2d8e74a001" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 972.628042] env[62499]: DEBUG nova.compute.manager [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 972.628042] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 972.628042] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3ab5cdb4-53ce-42ac-b3d2-a7544dc0fdf8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.638389] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0eee4c9d-0d97-4567-80f5-ca8819038452 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.651702] env[62499]: DEBUG nova.scheduler.client.report [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 972.661366] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d9151031-e57a-4bc9-9f6f-ce2d8e74a001 could not be found. [ 972.661628] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 972.662021] env[62499]: INFO nova.compute.manager [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Took 0.04 seconds to destroy the instance on the hypervisor. [ 972.662180] env[62499]: DEBUG oslo.service.loopingcall [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 972.662803] env[62499]: DEBUG nova.compute.manager [-] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 972.662803] env[62499]: DEBUG nova.network.neutron [-] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 972.676473] env[62499]: DEBUG nova.network.neutron [-] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 972.970247] env[62499]: DEBUG nova.compute.manager [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 972.995970] env[62499]: DEBUG nova.virt.hardware [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 972.996236] env[62499]: DEBUG nova.virt.hardware [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 972.996399] env[62499]: DEBUG nova.virt.hardware [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 972.996578] env[62499]: DEBUG nova.virt.hardware [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 972.996725] env[62499]: DEBUG nova.virt.hardware [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 972.996872] env[62499]: DEBUG nova.virt.hardware [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 972.997093] env[62499]: DEBUG nova.virt.hardware [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 972.997261] env[62499]: DEBUG nova.virt.hardware [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 972.997427] env[62499]: DEBUG nova.virt.hardware [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 972.997586] env[62499]: DEBUG nova.virt.hardware [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 972.997753] env[62499]: DEBUG nova.virt.hardware [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 972.998711] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-62f190d9-4d43-49bd-97b0-3a5107b0da09 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 973.006718] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f31271d8-ab48-4d22-94ac-d55337cbb37a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 973.019784] env[62499]: ERROR nova.compute.manager [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 9d32fadd-a63a-4b82-a49c-297505381d88, please check neutron logs for more information. [ 973.019784] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Traceback (most recent call last): [ 973.019784] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 973.019784] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] yield resources [ 973.019784] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 973.019784] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] self.driver.spawn(context, instance, image_meta, [ 973.019784] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 973.019784] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] self._vmops.spawn(context, instance, image_meta, injected_files, [ 973.019784] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 973.019784] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] vm_ref = self.build_virtual_machine(instance, [ 973.019784] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 973.020418] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] vif_infos = vmwarevif.get_vif_info(self._session, [ 973.020418] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 973.020418] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] for vif in network_info: [ 973.020418] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 973.020418] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] return self._sync_wrapper(fn, *args, **kwargs) [ 973.020418] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 973.020418] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] self.wait() [ 973.020418] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 973.020418] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] self[:] = self._gt.wait() [ 973.020418] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 973.020418] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] return self._exit_event.wait() [ 973.020418] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 973.020418] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] current.throw(*self._exc) [ 973.021013] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 973.021013] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] result = function(*args, **kwargs) [ 973.021013] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 973.021013] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] return func(*args, **kwargs) [ 973.021013] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 973.021013] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] raise e [ 973.021013] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 973.021013] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] nwinfo = self.network_api.allocate_for_instance( [ 973.021013] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 973.021013] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] created_port_ids = self._update_ports_for_instance( [ 973.021013] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 973.021013] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] with excutils.save_and_reraise_exception(): [ 973.021013] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 973.021594] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] self.force_reraise() [ 973.021594] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 973.021594] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] raise self.value [ 973.021594] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 973.021594] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] updated_port = self._update_port( [ 973.021594] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 973.021594] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] _ensure_no_port_binding_failure(port) [ 973.021594] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 973.021594] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] raise exception.PortBindingFailed(port_id=port['id']) [ 973.021594] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] nova.exception.PortBindingFailed: Binding failed for port 9d32fadd-a63a-4b82-a49c-297505381d88, please check neutron logs for more information. [ 973.021594] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] [ 973.021594] env[62499]: INFO nova.compute.manager [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Terminating instance [ 973.022173] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "refresh_cache-1c1379c2-348b-4be8-9415-594ce53ebf55" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 973.022173] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquired lock "refresh_cache-1c1379c2-348b-4be8-9415-594ce53ebf55" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 973.022341] env[62499]: DEBUG nova.network.neutron [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 973.114050] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 973.156815] env[62499]: DEBUG oslo_concurrency.lockutils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.210s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 973.157553] env[62499]: DEBUG nova.compute.manager [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 973.161264] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 6.539s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 973.178983] env[62499]: DEBUG nova.network.neutron [-] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 973.541490] env[62499]: DEBUG nova.network.neutron [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 973.629487] env[62499]: DEBUG nova.network.neutron [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 973.665522] env[62499]: DEBUG nova.compute.utils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 973.667525] env[62499]: DEBUG nova.compute.manager [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 973.667525] env[62499]: DEBUG nova.network.neutron [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 973.683014] env[62499]: INFO nova.compute.manager [-] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Took 1.02 seconds to deallocate network for instance. [ 973.684968] env[62499]: DEBUG nova.compute.claims [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 973.685158] env[62499]: DEBUG oslo_concurrency.lockutils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 973.718959] env[62499]: DEBUG nova.policy [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7da3078e8bb5444c96e007cfc2d57952', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c9f95d7e954f4413b01aad519c498967', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 973.956652] env[62499]: DEBUG nova.network.neutron [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Successfully created port: 613001b4-83a5-4226-a98a-9814c2d78697 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 974.132607] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Releasing lock "refresh_cache-1c1379c2-348b-4be8-9415-594ce53ebf55" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 974.133059] env[62499]: DEBUG nova.compute.manager [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 974.133265] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 974.133567] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-64d1c0d9-6c58-45e3-acfb-19549a7ba5e8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 974.143437] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eeb2506d-ffe8-4140-bf68-30c611b14ec9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 974.167296] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1c1379c2-348b-4be8-9415-594ce53ebf55 could not be found. [ 974.167534] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 974.167717] env[62499]: INFO nova.compute.manager [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Took 0.03 seconds to destroy the instance on the hypervisor. [ 974.167962] env[62499]: DEBUG oslo.service.loopingcall [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 974.168195] env[62499]: DEBUG nova.compute.manager [-] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 974.168290] env[62499]: DEBUG nova.network.neutron [-] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 974.172563] env[62499]: DEBUG nova.compute.manager [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 974.190041] env[62499]: DEBUG nova.network.neutron [-] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 974.199176] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance b4e185e6-b177-4212-bad8-b9fd1b51ea95 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 974.199432] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 636d27e5-f184-4656-9225-09c6ec0f0772 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 974.199563] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance d9151031-e57a-4bc9-9f6f-ce2d8e74a001 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 974.199691] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 1c1379c2-348b-4be8-9415-594ce53ebf55 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 974.199848] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 7d91c659-d0ac-435c-af73-d6629bcde6bf actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 974.414283] env[62499]: DEBUG nova.compute.manager [req-9f213992-b21e-4a12-9a2a-c6cccb06157c req-6712b571-dd95-4adb-87cc-e1e00fabb4a2 service nova] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Received event network-changed-9d32fadd-a63a-4b82-a49c-297505381d88 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 974.414503] env[62499]: DEBUG nova.compute.manager [req-9f213992-b21e-4a12-9a2a-c6cccb06157c req-6712b571-dd95-4adb-87cc-e1e00fabb4a2 service nova] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Refreshing instance network info cache due to event network-changed-9d32fadd-a63a-4b82-a49c-297505381d88. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 974.414758] env[62499]: DEBUG oslo_concurrency.lockutils [req-9f213992-b21e-4a12-9a2a-c6cccb06157c req-6712b571-dd95-4adb-87cc-e1e00fabb4a2 service nova] Acquiring lock "refresh_cache-1c1379c2-348b-4be8-9415-594ce53ebf55" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 974.414844] env[62499]: DEBUG oslo_concurrency.lockutils [req-9f213992-b21e-4a12-9a2a-c6cccb06157c req-6712b571-dd95-4adb-87cc-e1e00fabb4a2 service nova] Acquired lock "refresh_cache-1c1379c2-348b-4be8-9415-594ce53ebf55" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 974.414999] env[62499]: DEBUG nova.network.neutron [req-9f213992-b21e-4a12-9a2a-c6cccb06157c req-6712b571-dd95-4adb-87cc-e1e00fabb4a2 service nova] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Refreshing network info cache for port 9d32fadd-a63a-4b82-a49c-297505381d88 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 974.692875] env[62499]: DEBUG nova.network.neutron [-] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 974.705022] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 974.705022] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Total usable vcpus: 48, total allocated vcpus: 5 {{(pid=62499) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 974.705022] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1472MB phys_disk=200GB used_disk=5GB total_vcpus=48 used_vcpus=5 pci_stats=[] {{(pid=62499) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 974.759832] env[62499]: ERROR nova.compute.manager [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 613001b4-83a5-4226-a98a-9814c2d78697, please check neutron logs for more information. [ 974.759832] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 974.759832] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 974.759832] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 974.759832] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 974.759832] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 974.759832] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 974.759832] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 974.759832] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 974.759832] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 974.759832] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 974.759832] env[62499]: ERROR nova.compute.manager raise self.value [ 974.759832] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 974.759832] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 974.759832] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 974.759832] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 974.761288] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 974.761288] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 974.761288] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 613001b4-83a5-4226-a98a-9814c2d78697, please check neutron logs for more information. [ 974.761288] env[62499]: ERROR nova.compute.manager [ 974.761288] env[62499]: Traceback (most recent call last): [ 974.761288] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 974.761288] env[62499]: listener.cb(fileno) [ 974.761288] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 974.761288] env[62499]: result = function(*args, **kwargs) [ 974.761288] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 974.761288] env[62499]: return func(*args, **kwargs) [ 974.761288] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 974.761288] env[62499]: raise e [ 974.761288] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 974.761288] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 974.761288] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 974.761288] env[62499]: created_port_ids = self._update_ports_for_instance( [ 974.761288] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 974.761288] env[62499]: with excutils.save_and_reraise_exception(): [ 974.761288] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 974.761288] env[62499]: self.force_reraise() [ 974.761288] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 974.761288] env[62499]: raise self.value [ 974.761288] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 974.761288] env[62499]: updated_port = self._update_port( [ 974.761288] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 974.761288] env[62499]: _ensure_no_port_binding_failure(port) [ 974.761288] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 974.761288] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 974.762731] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 613001b4-83a5-4226-a98a-9814c2d78697, please check neutron logs for more information. [ 974.762731] env[62499]: Removing descriptor: 17 [ 974.796298] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb1476de-aa0d-41a4-a3e9-224b36362e7e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 974.804160] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-026bd100-4e79-49d0-9911-a9c543a44fcc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 974.832902] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c0a95d6-ce3d-4c7f-bece-dd35f9e6cf59 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 974.839407] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ce447390-0430-425b-83a4-d74f99b5a48f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 974.852895] env[62499]: DEBUG nova.compute.provider_tree [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 974.933683] env[62499]: DEBUG nova.network.neutron [req-9f213992-b21e-4a12-9a2a-c6cccb06157c req-6712b571-dd95-4adb-87cc-e1e00fabb4a2 service nova] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 975.013275] env[62499]: DEBUG nova.network.neutron [req-9f213992-b21e-4a12-9a2a-c6cccb06157c req-6712b571-dd95-4adb-87cc-e1e00fabb4a2 service nova] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 975.185251] env[62499]: DEBUG nova.compute.manager [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 975.194817] env[62499]: INFO nova.compute.manager [-] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Took 1.03 seconds to deallocate network for instance. [ 975.197115] env[62499]: DEBUG nova.compute.claims [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 975.197305] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 975.209507] env[62499]: DEBUG nova.virt.hardware [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 975.209759] env[62499]: DEBUG nova.virt.hardware [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 975.209924] env[62499]: DEBUG nova.virt.hardware [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 975.210178] env[62499]: DEBUG nova.virt.hardware [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 975.210330] env[62499]: DEBUG nova.virt.hardware [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 975.210478] env[62499]: DEBUG nova.virt.hardware [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 975.210676] env[62499]: DEBUG nova.virt.hardware [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 975.210831] env[62499]: DEBUG nova.virt.hardware [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 975.211011] env[62499]: DEBUG nova.virt.hardware [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 975.211229] env[62499]: DEBUG nova.virt.hardware [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 975.211415] env[62499]: DEBUG nova.virt.hardware [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 975.212257] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9914f83-9c45-4cc7-a6f9-b616e4b4ccfb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 975.220208] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2276f703-2b09-4586-ada4-373f1d6a8a9c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 975.233793] env[62499]: ERROR nova.compute.manager [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 613001b4-83a5-4226-a98a-9814c2d78697, please check neutron logs for more information. [ 975.233793] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Traceback (most recent call last): [ 975.233793] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 975.233793] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] yield resources [ 975.233793] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 975.233793] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] self.driver.spawn(context, instance, image_meta, [ 975.233793] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 975.233793] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] self._vmops.spawn(context, instance, image_meta, injected_files, [ 975.233793] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 975.233793] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] vm_ref = self.build_virtual_machine(instance, [ 975.233793] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 975.234261] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] vif_infos = vmwarevif.get_vif_info(self._session, [ 975.234261] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 975.234261] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] for vif in network_info: [ 975.234261] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 975.234261] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] return self._sync_wrapper(fn, *args, **kwargs) [ 975.234261] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 975.234261] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] self.wait() [ 975.234261] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 975.234261] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] self[:] = self._gt.wait() [ 975.234261] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 975.234261] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] return self._exit_event.wait() [ 975.234261] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 975.234261] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] current.throw(*self._exc) [ 975.234652] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 975.234652] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] result = function(*args, **kwargs) [ 975.234652] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 975.234652] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] return func(*args, **kwargs) [ 975.234652] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 975.234652] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] raise e [ 975.234652] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 975.234652] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] nwinfo = self.network_api.allocate_for_instance( [ 975.234652] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 975.234652] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] created_port_ids = self._update_ports_for_instance( [ 975.234652] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 975.234652] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] with excutils.save_and_reraise_exception(): [ 975.234652] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 975.235051] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] self.force_reraise() [ 975.235051] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 975.235051] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] raise self.value [ 975.235051] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 975.235051] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] updated_port = self._update_port( [ 975.235051] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 975.235051] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] _ensure_no_port_binding_failure(port) [ 975.235051] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 975.235051] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] raise exception.PortBindingFailed(port_id=port['id']) [ 975.235051] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] nova.exception.PortBindingFailed: Binding failed for port 613001b4-83a5-4226-a98a-9814c2d78697, please check neutron logs for more information. [ 975.235051] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] [ 975.235051] env[62499]: INFO nova.compute.manager [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Terminating instance [ 975.236321] env[62499]: DEBUG oslo_concurrency.lockutils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "refresh_cache-7d91c659-d0ac-435c-af73-d6629bcde6bf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 975.236482] env[62499]: DEBUG oslo_concurrency.lockutils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquired lock "refresh_cache-7d91c659-d0ac-435c-af73-d6629bcde6bf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 975.236649] env[62499]: DEBUG nova.network.neutron [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 975.355583] env[62499]: DEBUG nova.scheduler.client.report [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 975.516387] env[62499]: DEBUG oslo_concurrency.lockutils [req-9f213992-b21e-4a12-9a2a-c6cccb06157c req-6712b571-dd95-4adb-87cc-e1e00fabb4a2 service nova] Releasing lock "refresh_cache-1c1379c2-348b-4be8-9415-594ce53ebf55" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 975.516618] env[62499]: DEBUG nova.compute.manager [req-9f213992-b21e-4a12-9a2a-c6cccb06157c req-6712b571-dd95-4adb-87cc-e1e00fabb4a2 service nova] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Received event network-vif-deleted-9d32fadd-a63a-4b82-a49c-297505381d88 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 975.753996] env[62499]: DEBUG nova.network.neutron [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 975.832390] env[62499]: DEBUG nova.network.neutron [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 975.860815] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62499) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 975.861149] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.700s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 975.861520] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 8.387s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 976.338020] env[62499]: DEBUG oslo_concurrency.lockutils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Releasing lock "refresh_cache-7d91c659-d0ac-435c-af73-d6629bcde6bf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 976.338020] env[62499]: DEBUG nova.compute.manager [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 976.338020] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 976.338020] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f94c5930-081a-47ae-a0a5-aad1dc3ffd11 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 976.346019] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-176691f9-0f2b-4ca6-a52c-4d0180c12aea {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 976.369100] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7d91c659-d0ac-435c-af73-d6629bcde6bf could not be found. [ 976.369437] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 976.369723] env[62499]: INFO nova.compute.manager [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Took 0.03 seconds to destroy the instance on the hypervisor. [ 976.370095] env[62499]: DEBUG oslo.service.loopingcall [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 976.371312] env[62499]: DEBUG nova.compute.manager [-] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 976.371312] env[62499]: DEBUG nova.network.neutron [-] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 976.386255] env[62499]: DEBUG nova.network.neutron [-] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 976.439825] env[62499]: DEBUG nova.compute.manager [req-1a7465f5-762c-4bb8-930e-b0bc94769e34 req-b3afacb3-0f31-48a5-8bb7-660a4f4185e1 service nova] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Received event network-changed-613001b4-83a5-4226-a98a-9814c2d78697 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 976.440053] env[62499]: DEBUG nova.compute.manager [req-1a7465f5-762c-4bb8-930e-b0bc94769e34 req-b3afacb3-0f31-48a5-8bb7-660a4f4185e1 service nova] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Refreshing instance network info cache due to event network-changed-613001b4-83a5-4226-a98a-9814c2d78697. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 976.440266] env[62499]: DEBUG oslo_concurrency.lockutils [req-1a7465f5-762c-4bb8-930e-b0bc94769e34 req-b3afacb3-0f31-48a5-8bb7-660a4f4185e1 service nova] Acquiring lock "refresh_cache-7d91c659-d0ac-435c-af73-d6629bcde6bf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 976.440414] env[62499]: DEBUG oslo_concurrency.lockutils [req-1a7465f5-762c-4bb8-930e-b0bc94769e34 req-b3afacb3-0f31-48a5-8bb7-660a4f4185e1 service nova] Acquired lock "refresh_cache-7d91c659-d0ac-435c-af73-d6629bcde6bf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 976.440576] env[62499]: DEBUG nova.network.neutron [req-1a7465f5-762c-4bb8-930e-b0bc94769e34 req-b3afacb3-0f31-48a5-8bb7-660a4f4185e1 service nova] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Refreshing network info cache for port 613001b4-83a5-4226-a98a-9814c2d78697 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 976.446993] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-51208d20-a71c-400a-848d-8d7c8a7d3a1d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 976.455087] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39d9a468-13f3-4a41-9bfa-c0cef4a2dde4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 976.484812] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b2500b3-394b-4264-b348-d579c162c706 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 976.491304] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f6a2a54-0d00-4fe6-ba8f-0a30b231a914 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 976.503869] env[62499]: DEBUG nova.compute.provider_tree [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 976.888309] env[62499]: DEBUG nova.network.neutron [-] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 976.958262] env[62499]: DEBUG nova.network.neutron [req-1a7465f5-762c-4bb8-930e-b0bc94769e34 req-b3afacb3-0f31-48a5-8bb7-660a4f4185e1 service nova] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 977.007087] env[62499]: DEBUG nova.scheduler.client.report [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 977.025342] env[62499]: DEBUG nova.network.neutron [req-1a7465f5-762c-4bb8-930e-b0bc94769e34 req-b3afacb3-0f31-48a5-8bb7-660a4f4185e1 service nova] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 977.390735] env[62499]: INFO nova.compute.manager [-] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Took 1.02 seconds to deallocate network for instance. [ 977.393160] env[62499]: DEBUG nova.compute.claims [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 977.393350] env[62499]: DEBUG oslo_concurrency.lockutils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 977.512061] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.650s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 977.512697] env[62499]: ERROR nova.compute.manager [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 92271e85-7991-4870-b66b-67bbb7cfdd5c, please check neutron logs for more information. [ 977.512697] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Traceback (most recent call last): [ 977.512697] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 977.512697] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] self.driver.spawn(context, instance, image_meta, [ 977.512697] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 977.512697] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] self._vmops.spawn(context, instance, image_meta, injected_files, [ 977.512697] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 977.512697] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] vm_ref = self.build_virtual_machine(instance, [ 977.512697] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 977.512697] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] vif_infos = vmwarevif.get_vif_info(self._session, [ 977.512697] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 977.513201] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] for vif in network_info: [ 977.513201] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 977.513201] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] return self._sync_wrapper(fn, *args, **kwargs) [ 977.513201] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 977.513201] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] self.wait() [ 977.513201] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 977.513201] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] self[:] = self._gt.wait() [ 977.513201] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 977.513201] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] return self._exit_event.wait() [ 977.513201] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 977.513201] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] current.throw(*self._exc) [ 977.513201] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 977.513201] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] result = function(*args, **kwargs) [ 977.513742] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 977.513742] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] return func(*args, **kwargs) [ 977.513742] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 977.513742] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] raise e [ 977.513742] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 977.513742] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] nwinfo = self.network_api.allocate_for_instance( [ 977.513742] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 977.513742] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] created_port_ids = self._update_ports_for_instance( [ 977.513742] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 977.513742] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] with excutils.save_and_reraise_exception(): [ 977.513742] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 977.513742] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] self.force_reraise() [ 977.513742] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 977.514200] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] raise self.value [ 977.514200] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 977.514200] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] updated_port = self._update_port( [ 977.514200] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 977.514200] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] _ensure_no_port_binding_failure(port) [ 977.514200] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 977.514200] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] raise exception.PortBindingFailed(port_id=port['id']) [ 977.514200] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] nova.exception.PortBindingFailed: Binding failed for port 92271e85-7991-4870-b66b-67bbb7cfdd5c, please check neutron logs for more information. [ 977.514200] env[62499]: ERROR nova.compute.manager [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] [ 977.514200] env[62499]: DEBUG nova.compute.utils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Binding failed for port 92271e85-7991-4870-b66b-67bbb7cfdd5c, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 977.514534] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 8.279s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 977.517224] env[62499]: DEBUG nova.compute.manager [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Build of instance 636d27e5-f184-4656-9225-09c6ec0f0772 was re-scheduled: Binding failed for port 92271e85-7991-4870-b66b-67bbb7cfdd5c, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 977.517627] env[62499]: DEBUG nova.compute.manager [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 977.517844] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Acquiring lock "refresh_cache-636d27e5-f184-4656-9225-09c6ec0f0772" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 977.518031] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Acquired lock "refresh_cache-636d27e5-f184-4656-9225-09c6ec0f0772" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 977.518192] env[62499]: DEBUG nova.network.neutron [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 977.527338] env[62499]: DEBUG oslo_concurrency.lockutils [req-1a7465f5-762c-4bb8-930e-b0bc94769e34 req-b3afacb3-0f31-48a5-8bb7-660a4f4185e1 service nova] Releasing lock "refresh_cache-7d91c659-d0ac-435c-af73-d6629bcde6bf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 977.527560] env[62499]: DEBUG nova.compute.manager [req-1a7465f5-762c-4bb8-930e-b0bc94769e34 req-b3afacb3-0f31-48a5-8bb7-660a4f4185e1 service nova] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Received event network-vif-deleted-613001b4-83a5-4226-a98a-9814c2d78697 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 978.038770] env[62499]: DEBUG nova.network.neutron [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 978.108444] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5e2e5a4-d6e3-4d7e-98dd-5650f344a5ac {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 978.112400] env[62499]: DEBUG nova.network.neutron [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 978.118366] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe27da18-6e33-446e-847f-aa63836341e7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 978.149555] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e18e900-5cad-453c-8f6d-380acc628380 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 978.156285] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-475565d4-5e44-4530-8368-094bb5624511 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 978.169162] env[62499]: DEBUG nova.compute.provider_tree [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 978.614922] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Releasing lock "refresh_cache-636d27e5-f184-4656-9225-09c6ec0f0772" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 978.615321] env[62499]: DEBUG nova.compute.manager [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 978.615321] env[62499]: DEBUG nova.compute.manager [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 978.615487] env[62499]: DEBUG nova.network.neutron [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 978.630651] env[62499]: DEBUG nova.network.neutron [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 978.672146] env[62499]: DEBUG nova.scheduler.client.report [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 979.133159] env[62499]: DEBUG nova.network.neutron [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 979.176933] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.662s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 979.177910] env[62499]: ERROR nova.compute.manager [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b92e7d6b-5c1d-4570-ab7e-15c081e63812, please check neutron logs for more information. [ 979.177910] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Traceback (most recent call last): [ 979.177910] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 979.177910] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] self.driver.spawn(context, instance, image_meta, [ 979.177910] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 979.177910] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] self._vmops.spawn(context, instance, image_meta, injected_files, [ 979.177910] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 979.177910] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] vm_ref = self.build_virtual_machine(instance, [ 979.177910] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 979.177910] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] vif_infos = vmwarevif.get_vif_info(self._session, [ 979.177910] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 979.178335] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] for vif in network_info: [ 979.178335] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 979.178335] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] return self._sync_wrapper(fn, *args, **kwargs) [ 979.178335] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 979.178335] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] self.wait() [ 979.178335] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 979.178335] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] self[:] = self._gt.wait() [ 979.178335] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 979.178335] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] return self._exit_event.wait() [ 979.178335] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 979.178335] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] current.throw(*self._exc) [ 979.178335] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 979.178335] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] result = function(*args, **kwargs) [ 979.178771] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 979.178771] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] return func(*args, **kwargs) [ 979.178771] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 979.178771] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] raise e [ 979.178771] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 979.178771] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] nwinfo = self.network_api.allocate_for_instance( [ 979.178771] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 979.178771] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] created_port_ids = self._update_ports_for_instance( [ 979.178771] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 979.178771] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] with excutils.save_and_reraise_exception(): [ 979.178771] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 979.178771] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] self.force_reraise() [ 979.178771] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 979.179206] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] raise self.value [ 979.179206] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 979.179206] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] updated_port = self._update_port( [ 979.179206] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 979.179206] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] _ensure_no_port_binding_failure(port) [ 979.179206] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 979.179206] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] raise exception.PortBindingFailed(port_id=port['id']) [ 979.179206] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] nova.exception.PortBindingFailed: Binding failed for port b92e7d6b-5c1d-4570-ab7e-15c081e63812, please check neutron logs for more information. [ 979.179206] env[62499]: ERROR nova.compute.manager [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] [ 979.179206] env[62499]: DEBUG nova.compute.utils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Binding failed for port b92e7d6b-5c1d-4570-ab7e-15c081e63812, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 979.179710] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 6.066s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 979.181183] env[62499]: INFO nova.compute.claims [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 979.183628] env[62499]: DEBUG nova.compute.manager [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Build of instance b4e185e6-b177-4212-bad8-b9fd1b51ea95 was re-scheduled: Binding failed for port b92e7d6b-5c1d-4570-ab7e-15c081e63812, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 979.184054] env[62499]: DEBUG nova.compute.manager [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 979.184271] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquiring lock "refresh_cache-b4e185e6-b177-4212-bad8-b9fd1b51ea95" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 979.184420] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Acquired lock "refresh_cache-b4e185e6-b177-4212-bad8-b9fd1b51ea95" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 979.184575] env[62499]: DEBUG nova.network.neutron [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 979.635793] env[62499]: INFO nova.compute.manager [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] [instance: 636d27e5-f184-4656-9225-09c6ec0f0772] Took 1.02 seconds to deallocate network for instance. [ 979.702890] env[62499]: DEBUG nova.network.neutron [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 979.768884] env[62499]: DEBUG nova.network.neutron [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 980.272608] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Releasing lock "refresh_cache-b4e185e6-b177-4212-bad8-b9fd1b51ea95" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 980.272849] env[62499]: DEBUG nova.compute.manager [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 980.273041] env[62499]: DEBUG nova.compute.manager [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 980.273224] env[62499]: DEBUG nova.network.neutron [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 980.286507] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85ed25ea-2848-4842-a704-001fa82b9384 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.290059] env[62499]: DEBUG nova.network.neutron [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 980.297475] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3115cebe-f5f7-43bc-bc1d-46af0e3ba952 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.329015] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f7ec3e9-90cb-4cd0-a43e-7e4922066b54 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.337455] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85969b6f-874d-4518-a9cc-2e56ef8d3c2a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.351619] env[62499]: DEBUG nova.compute.provider_tree [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 980.664566] env[62499]: INFO nova.scheduler.client.report [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Deleted allocations for instance 636d27e5-f184-4656-9225-09c6ec0f0772 [ 980.793639] env[62499]: DEBUG nova.network.neutron [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 980.854844] env[62499]: DEBUG nova.scheduler.client.report [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 981.172718] env[62499]: DEBUG oslo_concurrency.lockutils [None req-f2c542da-4b8d-471b-be4d-3a37acc71230 tempest-ServerMetadataTestJSON-949477722 tempest-ServerMetadataTestJSON-949477722-project-member] Lock "636d27e5-f184-4656-9225-09c6ec0f0772" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 29.078s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 981.296966] env[62499]: INFO nova.compute.manager [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] [instance: b4e185e6-b177-4212-bad8-b9fd1b51ea95] Took 1.02 seconds to deallocate network for instance. [ 981.360040] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.180s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 981.360529] env[62499]: DEBUG nova.compute.manager [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 981.364391] env[62499]: DEBUG oslo_concurrency.lockutils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 7.678s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 981.872045] env[62499]: DEBUG nova.compute.utils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 981.876652] env[62499]: DEBUG nova.compute.manager [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 981.876652] env[62499]: DEBUG nova.network.neutron [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 981.919074] env[62499]: DEBUG nova.policy [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '02640299eb9c4a65bc80de6242dd2cd5', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f509dfe537d441bcacb1a4aab3607214', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 981.952785] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92d47c75-f1a6-455b-9cdd-29ae95076a6f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 981.960563] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-88b26150-2f5d-491b-b6a6-27f4aa27ae2c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 981.991945] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20e87682-8213-4ff3-991f-7984ab057cc6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 981.999298] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fadbd4ba-b210-46c2-987e-f0ee2afbaf5d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 982.012835] env[62499]: DEBUG nova.compute.provider_tree [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 982.191905] env[62499]: DEBUG nova.network.neutron [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Successfully created port: e1b1c46e-1478-45a2-9e15-0b8ddcd44cb4 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 982.336104] env[62499]: INFO nova.scheduler.client.report [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Deleted allocations for instance b4e185e6-b177-4212-bad8-b9fd1b51ea95 [ 982.374471] env[62499]: DEBUG nova.compute.manager [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 982.516322] env[62499]: DEBUG nova.scheduler.client.report [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 982.848104] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67f41cfa-a853-430d-aa7a-71084176d495 tempest-ImagesTestJSON-1154998582 tempest-ImagesTestJSON-1154998582-project-member] Lock "b4e185e6-b177-4212-bad8-b9fd1b51ea95" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 30.753s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 982.982448] env[62499]: DEBUG nova.compute.manager [req-017ec0fb-e44c-4a6c-8a14-d226995a6a1e req-018bb73d-4070-4178-9146-1e3b01f90f8c service nova] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Received event network-changed-e1b1c46e-1478-45a2-9e15-0b8ddcd44cb4 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 982.982706] env[62499]: DEBUG nova.compute.manager [req-017ec0fb-e44c-4a6c-8a14-d226995a6a1e req-018bb73d-4070-4178-9146-1e3b01f90f8c service nova] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Refreshing instance network info cache due to event network-changed-e1b1c46e-1478-45a2-9e15-0b8ddcd44cb4. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 982.982871] env[62499]: DEBUG oslo_concurrency.lockutils [req-017ec0fb-e44c-4a6c-8a14-d226995a6a1e req-018bb73d-4070-4178-9146-1e3b01f90f8c service nova] Acquiring lock "refresh_cache-fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 982.984623] env[62499]: DEBUG oslo_concurrency.lockutils [req-017ec0fb-e44c-4a6c-8a14-d226995a6a1e req-018bb73d-4070-4178-9146-1e3b01f90f8c service nova] Acquired lock "refresh_cache-fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 982.984623] env[62499]: DEBUG nova.network.neutron [req-017ec0fb-e44c-4a6c-8a14-d226995a6a1e req-018bb73d-4070-4178-9146-1e3b01f90f8c service nova] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Refreshing network info cache for port e1b1c46e-1478-45a2-9e15-0b8ddcd44cb4 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 983.021633] env[62499]: DEBUG oslo_concurrency.lockutils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.658s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 983.022271] env[62499]: ERROR nova.compute.manager [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 84cefa2b-4e20-45df-a4c6-1b623a06142c, please check neutron logs for more information. [ 983.022271] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Traceback (most recent call last): [ 983.022271] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 983.022271] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] self.driver.spawn(context, instance, image_meta, [ 983.022271] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 983.022271] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] self._vmops.spawn(context, instance, image_meta, injected_files, [ 983.022271] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 983.022271] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] vm_ref = self.build_virtual_machine(instance, [ 983.022271] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 983.022271] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] vif_infos = vmwarevif.get_vif_info(self._session, [ 983.022271] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 983.022790] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] for vif in network_info: [ 983.022790] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 983.022790] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] return self._sync_wrapper(fn, *args, **kwargs) [ 983.022790] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 983.022790] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] self.wait() [ 983.022790] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 983.022790] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] self[:] = self._gt.wait() [ 983.022790] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 983.022790] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] return self._exit_event.wait() [ 983.022790] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 983.022790] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] current.throw(*self._exc) [ 983.022790] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 983.022790] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] result = function(*args, **kwargs) [ 983.023171] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 983.023171] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] return func(*args, **kwargs) [ 983.023171] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 983.023171] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] raise e [ 983.023171] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 983.023171] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] nwinfo = self.network_api.allocate_for_instance( [ 983.023171] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 983.023171] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] created_port_ids = self._update_ports_for_instance( [ 983.023171] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 983.023171] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] with excutils.save_and_reraise_exception(): [ 983.023171] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 983.023171] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] self.force_reraise() [ 983.023171] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 983.023772] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] raise self.value [ 983.023772] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 983.023772] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] updated_port = self._update_port( [ 983.023772] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 983.023772] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] _ensure_no_port_binding_failure(port) [ 983.023772] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 983.023772] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] raise exception.PortBindingFailed(port_id=port['id']) [ 983.023772] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] nova.exception.PortBindingFailed: Binding failed for port 84cefa2b-4e20-45df-a4c6-1b623a06142c, please check neutron logs for more information. [ 983.023772] env[62499]: ERROR nova.compute.manager [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] [ 983.023772] env[62499]: DEBUG nova.compute.utils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Binding failed for port 84cefa2b-4e20-45df-a4c6-1b623a06142c, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 983.024512] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 7.827s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 983.030431] env[62499]: DEBUG nova.compute.manager [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Build of instance d9151031-e57a-4bc9-9f6f-ce2d8e74a001 was re-scheduled: Binding failed for port 84cefa2b-4e20-45df-a4c6-1b623a06142c, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 983.030431] env[62499]: DEBUG nova.compute.manager [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 983.030431] env[62499]: DEBUG oslo_concurrency.lockutils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Acquiring lock "refresh_cache-d9151031-e57a-4bc9-9f6f-ce2d8e74a001" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 983.030431] env[62499]: DEBUG oslo_concurrency.lockutils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Acquired lock "refresh_cache-d9151031-e57a-4bc9-9f6f-ce2d8e74a001" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 983.030661] env[62499]: DEBUG nova.network.neutron [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 983.334562] env[62499]: ERROR nova.compute.manager [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port e1b1c46e-1478-45a2-9e15-0b8ddcd44cb4, please check neutron logs for more information. [ 983.334562] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 983.334562] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 983.334562] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 983.334562] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 983.334562] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 983.334562] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 983.334562] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 983.334562] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 983.334562] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 983.334562] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 983.334562] env[62499]: ERROR nova.compute.manager raise self.value [ 983.334562] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 983.334562] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 983.334562] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 983.334562] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 983.335318] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 983.335318] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 983.335318] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port e1b1c46e-1478-45a2-9e15-0b8ddcd44cb4, please check neutron logs for more information. [ 983.335318] env[62499]: ERROR nova.compute.manager [ 983.335318] env[62499]: Traceback (most recent call last): [ 983.335318] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 983.335318] env[62499]: listener.cb(fileno) [ 983.335318] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 983.335318] env[62499]: result = function(*args, **kwargs) [ 983.335318] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 983.335318] env[62499]: return func(*args, **kwargs) [ 983.335318] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 983.335318] env[62499]: raise e [ 983.335318] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 983.335318] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 983.335318] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 983.335318] env[62499]: created_port_ids = self._update_ports_for_instance( [ 983.335318] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 983.335318] env[62499]: with excutils.save_and_reraise_exception(): [ 983.335318] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 983.335318] env[62499]: self.force_reraise() [ 983.335318] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 983.335318] env[62499]: raise self.value [ 983.335318] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 983.335318] env[62499]: updated_port = self._update_port( [ 983.335318] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 983.335318] env[62499]: _ensure_no_port_binding_failure(port) [ 983.335318] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 983.335318] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 983.336429] env[62499]: nova.exception.PortBindingFailed: Binding failed for port e1b1c46e-1478-45a2-9e15-0b8ddcd44cb4, please check neutron logs for more information. [ 983.336429] env[62499]: Removing descriptor: 17 [ 983.385641] env[62499]: DEBUG nova.compute.manager [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 983.416416] env[62499]: DEBUG nova.virt.hardware [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 983.416668] env[62499]: DEBUG nova.virt.hardware [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 983.418463] env[62499]: DEBUG nova.virt.hardware [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 983.418463] env[62499]: DEBUG nova.virt.hardware [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 983.418463] env[62499]: DEBUG nova.virt.hardware [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 983.418463] env[62499]: DEBUG nova.virt.hardware [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 983.418463] env[62499]: DEBUG nova.virt.hardware [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 983.418463] env[62499]: DEBUG nova.virt.hardware [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 983.418759] env[62499]: DEBUG nova.virt.hardware [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 983.418759] env[62499]: DEBUG nova.virt.hardware [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 983.418759] env[62499]: DEBUG nova.virt.hardware [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 983.419888] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3d228d5-1a43-47c2-8fd9-60e9f83d92a1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 983.427086] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-102fad3d-0869-4d07-8730-523e66236c96 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 983.441180] env[62499]: ERROR nova.compute.manager [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port e1b1c46e-1478-45a2-9e15-0b8ddcd44cb4, please check neutron logs for more information. [ 983.441180] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Traceback (most recent call last): [ 983.441180] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 983.441180] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] yield resources [ 983.441180] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 983.441180] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] self.driver.spawn(context, instance, image_meta, [ 983.441180] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 983.441180] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] self._vmops.spawn(context, instance, image_meta, injected_files, [ 983.441180] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 983.441180] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] vm_ref = self.build_virtual_machine(instance, [ 983.441180] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 983.441573] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] vif_infos = vmwarevif.get_vif_info(self._session, [ 983.441573] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 983.441573] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] for vif in network_info: [ 983.441573] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 983.441573] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] return self._sync_wrapper(fn, *args, **kwargs) [ 983.441573] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 983.441573] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] self.wait() [ 983.441573] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 983.441573] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] self[:] = self._gt.wait() [ 983.441573] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 983.441573] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] return self._exit_event.wait() [ 983.441573] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 983.441573] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] current.throw(*self._exc) [ 983.441939] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 983.441939] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] result = function(*args, **kwargs) [ 983.441939] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 983.441939] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] return func(*args, **kwargs) [ 983.441939] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 983.441939] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] raise e [ 983.441939] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 983.441939] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] nwinfo = self.network_api.allocate_for_instance( [ 983.441939] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 983.441939] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] created_port_ids = self._update_ports_for_instance( [ 983.441939] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 983.441939] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] with excutils.save_and_reraise_exception(): [ 983.441939] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 983.442341] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] self.force_reraise() [ 983.442341] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 983.442341] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] raise self.value [ 983.442341] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 983.442341] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] updated_port = self._update_port( [ 983.442341] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 983.442341] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] _ensure_no_port_binding_failure(port) [ 983.442341] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 983.442341] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] raise exception.PortBindingFailed(port_id=port['id']) [ 983.442341] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] nova.exception.PortBindingFailed: Binding failed for port e1b1c46e-1478-45a2-9e15-0b8ddcd44cb4, please check neutron logs for more information. [ 983.442341] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] [ 983.442341] env[62499]: INFO nova.compute.manager [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Terminating instance [ 983.443566] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "refresh_cache-fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 983.502794] env[62499]: DEBUG nova.network.neutron [req-017ec0fb-e44c-4a6c-8a14-d226995a6a1e req-018bb73d-4070-4178-9146-1e3b01f90f8c service nova] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 983.556705] env[62499]: DEBUG nova.network.neutron [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 983.580778] env[62499]: DEBUG nova.network.neutron [req-017ec0fb-e44c-4a6c-8a14-d226995a6a1e req-018bb73d-4070-4178-9146-1e3b01f90f8c service nova] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 983.603667] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96dfc081-ed36-4b74-849b-43db5a0b40ce {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 983.612537] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab4f7dd8-311a-4002-a2f8-366a50117b53 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 983.645875] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38e8f40f-9a70-41a6-8ac0-aa857665cfcf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 983.652867] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2abb3ef7-27d3-4240-a6d3-4fb25a5a8091 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 983.665718] env[62499]: DEBUG nova.compute.provider_tree [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 983.712392] env[62499]: DEBUG nova.network.neutron [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 984.084634] env[62499]: DEBUG oslo_concurrency.lockutils [req-017ec0fb-e44c-4a6c-8a14-d226995a6a1e req-018bb73d-4070-4178-9146-1e3b01f90f8c service nova] Releasing lock "refresh_cache-fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 984.085071] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquired lock "refresh_cache-fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 984.085279] env[62499]: DEBUG nova.network.neutron [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 984.168514] env[62499]: DEBUG nova.scheduler.client.report [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 984.214626] env[62499]: DEBUG oslo_concurrency.lockutils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Releasing lock "refresh_cache-d9151031-e57a-4bc9-9f6f-ce2d8e74a001" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 984.214855] env[62499]: DEBUG nova.compute.manager [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 984.215052] env[62499]: DEBUG nova.compute.manager [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 984.215226] env[62499]: DEBUG nova.network.neutron [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 984.234767] env[62499]: DEBUG nova.network.neutron [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 984.603945] env[62499]: DEBUG nova.network.neutron [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 984.673079] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.648s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 984.673780] env[62499]: ERROR nova.compute.manager [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 9d32fadd-a63a-4b82-a49c-297505381d88, please check neutron logs for more information. [ 984.673780] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Traceback (most recent call last): [ 984.673780] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 984.673780] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] self.driver.spawn(context, instance, image_meta, [ 984.673780] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 984.673780] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] self._vmops.spawn(context, instance, image_meta, injected_files, [ 984.673780] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 984.673780] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] vm_ref = self.build_virtual_machine(instance, [ 984.673780] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 984.673780] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] vif_infos = vmwarevif.get_vif_info(self._session, [ 984.673780] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 984.674264] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] for vif in network_info: [ 984.674264] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 984.674264] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] return self._sync_wrapper(fn, *args, **kwargs) [ 984.674264] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 984.674264] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] self.wait() [ 984.674264] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 984.674264] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] self[:] = self._gt.wait() [ 984.674264] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 984.674264] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] return self._exit_event.wait() [ 984.674264] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 984.674264] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] current.throw(*self._exc) [ 984.674264] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 984.674264] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] result = function(*args, **kwargs) [ 984.674682] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 984.674682] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] return func(*args, **kwargs) [ 984.674682] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 984.674682] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] raise e [ 984.674682] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 984.674682] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] nwinfo = self.network_api.allocate_for_instance( [ 984.674682] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 984.674682] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] created_port_ids = self._update_ports_for_instance( [ 984.674682] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 984.674682] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] with excutils.save_and_reraise_exception(): [ 984.674682] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 984.674682] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] self.force_reraise() [ 984.674682] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 984.675135] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] raise self.value [ 984.675135] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 984.675135] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] updated_port = self._update_port( [ 984.675135] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 984.675135] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] _ensure_no_port_binding_failure(port) [ 984.675135] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 984.675135] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] raise exception.PortBindingFailed(port_id=port['id']) [ 984.675135] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] nova.exception.PortBindingFailed: Binding failed for port 9d32fadd-a63a-4b82-a49c-297505381d88, please check neutron logs for more information. [ 984.675135] env[62499]: ERROR nova.compute.manager [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] [ 984.675135] env[62499]: DEBUG nova.compute.utils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Binding failed for port 9d32fadd-a63a-4b82-a49c-297505381d88, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 984.675733] env[62499]: DEBUG oslo_concurrency.lockutils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 7.282s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 984.678529] env[62499]: DEBUG nova.compute.manager [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Build of instance 1c1379c2-348b-4be8-9415-594ce53ebf55 was re-scheduled: Binding failed for port 9d32fadd-a63a-4b82-a49c-297505381d88, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 984.678957] env[62499]: DEBUG nova.compute.manager [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 984.679196] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquiring lock "refresh_cache-1c1379c2-348b-4be8-9415-594ce53ebf55" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 984.679353] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Acquired lock "refresh_cache-1c1379c2-348b-4be8-9415-594ce53ebf55" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 984.679546] env[62499]: DEBUG nova.network.neutron [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 984.681535] env[62499]: DEBUG nova.network.neutron [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 984.736819] env[62499]: DEBUG nova.network.neutron [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 985.012490] env[62499]: DEBUG nova.compute.manager [req-ff8ab5aa-6468-4f9f-92d2-3a9249e63140 req-2b417b66-c40b-41ea-8db2-28821f84fc1f service nova] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Received event network-vif-deleted-e1b1c46e-1478-45a2-9e15-0b8ddcd44cb4 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 985.187566] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Releasing lock "refresh_cache-fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 985.187950] env[62499]: DEBUG nova.compute.manager [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 985.188207] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 985.188707] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9e496646-2f7f-42ae-8c5f-261d1540e553 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 985.197926] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93dac0d1-ac18-42b5-b18a-91c79a31e241 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 985.211804] env[62499]: DEBUG nova.network.neutron [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 985.224321] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab could not be found. [ 985.224537] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 985.224716] env[62499]: INFO nova.compute.manager [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Took 0.04 seconds to destroy the instance on the hypervisor. [ 985.224954] env[62499]: DEBUG oslo.service.loopingcall [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 985.227201] env[62499]: DEBUG nova.compute.manager [-] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 985.227303] env[62499]: DEBUG nova.network.neutron [-] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 985.238908] env[62499]: INFO nova.compute.manager [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: d9151031-e57a-4bc9-9f6f-ce2d8e74a001] Took 1.02 seconds to deallocate network for instance. [ 985.241926] env[62499]: DEBUG nova.network.neutron [-] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 985.263594] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e99f4eb-6cd7-4908-b238-f66e30ad2cb5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 985.271845] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f30ad8f-7390-4a26-a7ed-700b1426a21f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 985.300858] env[62499]: DEBUG nova.network.neutron [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 985.302315] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d4fa98e-eb0b-4055-922b-fcf009c95e5c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 985.309938] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-128c03ab-7eb3-44f4-b393-2d54c2f68101 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 985.322985] env[62499]: DEBUG nova.compute.provider_tree [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 985.747514] env[62499]: DEBUG nova.network.neutron [-] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 985.806145] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Releasing lock "refresh_cache-1c1379c2-348b-4be8-9415-594ce53ebf55" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 985.806381] env[62499]: DEBUG nova.compute.manager [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 985.806602] env[62499]: DEBUG nova.compute.manager [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 985.806777] env[62499]: DEBUG nova.network.neutron [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 985.821847] env[62499]: DEBUG nova.network.neutron [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 985.825995] env[62499]: DEBUG nova.scheduler.client.report [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 986.252759] env[62499]: INFO nova.compute.manager [-] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Took 1.03 seconds to deallocate network for instance. [ 986.255191] env[62499]: DEBUG nova.compute.claims [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 986.255419] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 986.268214] env[62499]: INFO nova.scheduler.client.report [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Deleted allocations for instance d9151031-e57a-4bc9-9f6f-ce2d8e74a001 [ 986.324217] env[62499]: DEBUG nova.network.neutron [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 986.330279] env[62499]: DEBUG oslo_concurrency.lockutils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.654s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 986.330861] env[62499]: ERROR nova.compute.manager [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 613001b4-83a5-4226-a98a-9814c2d78697, please check neutron logs for more information. [ 986.330861] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Traceback (most recent call last): [ 986.330861] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 986.330861] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] self.driver.spawn(context, instance, image_meta, [ 986.330861] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 986.330861] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] self._vmops.spawn(context, instance, image_meta, injected_files, [ 986.330861] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 986.330861] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] vm_ref = self.build_virtual_machine(instance, [ 986.330861] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 986.330861] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] vif_infos = vmwarevif.get_vif_info(self._session, [ 986.330861] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 986.331292] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] for vif in network_info: [ 986.331292] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 986.331292] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] return self._sync_wrapper(fn, *args, **kwargs) [ 986.331292] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 986.331292] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] self.wait() [ 986.331292] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 986.331292] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] self[:] = self._gt.wait() [ 986.331292] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 986.331292] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] return self._exit_event.wait() [ 986.331292] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 986.331292] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] current.throw(*self._exc) [ 986.331292] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 986.331292] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] result = function(*args, **kwargs) [ 986.331865] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 986.331865] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] return func(*args, **kwargs) [ 986.331865] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 986.331865] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] raise e [ 986.331865] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 986.331865] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] nwinfo = self.network_api.allocate_for_instance( [ 986.331865] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 986.331865] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] created_port_ids = self._update_ports_for_instance( [ 986.331865] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 986.331865] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] with excutils.save_and_reraise_exception(): [ 986.331865] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 986.331865] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] self.force_reraise() [ 986.331865] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 986.332452] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] raise self.value [ 986.332452] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 986.332452] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] updated_port = self._update_port( [ 986.332452] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 986.332452] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] _ensure_no_port_binding_failure(port) [ 986.332452] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 986.332452] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] raise exception.PortBindingFailed(port_id=port['id']) [ 986.332452] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] nova.exception.PortBindingFailed: Binding failed for port 613001b4-83a5-4226-a98a-9814c2d78697, please check neutron logs for more information. [ 986.332452] env[62499]: ERROR nova.compute.manager [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] [ 986.332452] env[62499]: DEBUG nova.compute.utils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Binding failed for port 613001b4-83a5-4226-a98a-9814c2d78697, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 986.332780] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.077s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 986.335595] env[62499]: DEBUG nova.compute.manager [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Build of instance 7d91c659-d0ac-435c-af73-d6629bcde6bf was re-scheduled: Binding failed for port 613001b4-83a5-4226-a98a-9814c2d78697, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 986.336018] env[62499]: DEBUG nova.compute.manager [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 986.336241] env[62499]: DEBUG oslo_concurrency.lockutils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "refresh_cache-7d91c659-d0ac-435c-af73-d6629bcde6bf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 986.336392] env[62499]: DEBUG oslo_concurrency.lockutils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquired lock "refresh_cache-7d91c659-d0ac-435c-af73-d6629bcde6bf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 986.336550] env[62499]: DEBUG nova.network.neutron [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 986.776673] env[62499]: DEBUG oslo_concurrency.lockutils [None req-57946dbc-1615-4927-b014-841c3deeb332 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Lock "d9151031-e57a-4bc9-9f6f-ce2d8e74a001" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 28.128s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 986.826625] env[62499]: INFO nova.compute.manager [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] [instance: 1c1379c2-348b-4be8-9415-594ce53ebf55] Took 1.02 seconds to deallocate network for instance. [ 986.857315] env[62499]: DEBUG nova.network.neutron [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 986.887144] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bb7d794f-8245-452d-9726-0303bd87c9c9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 986.897107] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6a904d9-966c-482c-bf5f-dab378c62559 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 986.927326] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3459235d-012f-499e-a995-32b40d26ca52 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 986.929662] env[62499]: DEBUG nova.network.neutron [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 986.935761] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef93fa65-fb32-4f1a-b7cc-ec78c051eee9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 986.949488] env[62499]: DEBUG nova.compute.provider_tree [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 987.432200] env[62499]: DEBUG oslo_concurrency.lockutils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Releasing lock "refresh_cache-7d91c659-d0ac-435c-af73-d6629bcde6bf" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 987.432561] env[62499]: DEBUG nova.compute.manager [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 987.432611] env[62499]: DEBUG nova.compute.manager [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 987.432774] env[62499]: DEBUG nova.network.neutron [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 987.452663] env[62499]: DEBUG nova.scheduler.client.report [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 987.456761] env[62499]: DEBUG nova.network.neutron [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 987.855087] env[62499]: INFO nova.scheduler.client.report [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Deleted allocations for instance 1c1379c2-348b-4be8-9415-594ce53ebf55 [ 987.927424] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 987.927737] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 987.960409] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.628s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 987.961082] env[62499]: ERROR nova.compute.manager [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port e1b1c46e-1478-45a2-9e15-0b8ddcd44cb4, please check neutron logs for more information. [ 987.961082] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Traceback (most recent call last): [ 987.961082] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 987.961082] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] self.driver.spawn(context, instance, image_meta, [ 987.961082] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 987.961082] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] self._vmops.spawn(context, instance, image_meta, injected_files, [ 987.961082] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 987.961082] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] vm_ref = self.build_virtual_machine(instance, [ 987.961082] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 987.961082] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] vif_infos = vmwarevif.get_vif_info(self._session, [ 987.961082] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 987.961465] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] for vif in network_info: [ 987.961465] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 987.961465] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] return self._sync_wrapper(fn, *args, **kwargs) [ 987.961465] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 987.961465] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] self.wait() [ 987.961465] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 987.961465] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] self[:] = self._gt.wait() [ 987.961465] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 987.961465] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] return self._exit_event.wait() [ 987.961465] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 987.961465] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] current.throw(*self._exc) [ 987.961465] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 987.961465] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] result = function(*args, **kwargs) [ 987.961880] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 987.961880] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] return func(*args, **kwargs) [ 987.961880] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 987.961880] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] raise e [ 987.961880] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 987.961880] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] nwinfo = self.network_api.allocate_for_instance( [ 987.961880] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 987.961880] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] created_port_ids = self._update_ports_for_instance( [ 987.961880] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 987.961880] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] with excutils.save_and_reraise_exception(): [ 987.961880] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 987.961880] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] self.force_reraise() [ 987.961880] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 987.962307] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] raise self.value [ 987.962307] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 987.962307] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] updated_port = self._update_port( [ 987.962307] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 987.962307] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] _ensure_no_port_binding_failure(port) [ 987.962307] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 987.962307] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] raise exception.PortBindingFailed(port_id=port['id']) [ 987.962307] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] nova.exception.PortBindingFailed: Binding failed for port e1b1c46e-1478-45a2-9e15-0b8ddcd44cb4, please check neutron logs for more information. [ 987.962307] env[62499]: ERROR nova.compute.manager [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] [ 987.962307] env[62499]: DEBUG nova.compute.utils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Binding failed for port e1b1c46e-1478-45a2-9e15-0b8ddcd44cb4, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 987.963050] env[62499]: DEBUG nova.network.neutron [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 987.964167] env[62499]: DEBUG nova.compute.manager [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Build of instance fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab was re-scheduled: Binding failed for port e1b1c46e-1478-45a2-9e15-0b8ddcd44cb4, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 987.964574] env[62499]: DEBUG nova.compute.manager [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 987.964791] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "refresh_cache-fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 987.964933] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquired lock "refresh_cache-fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 987.965147] env[62499]: DEBUG nova.network.neutron [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 988.363146] env[62499]: DEBUG oslo_concurrency.lockutils [None req-2348a2d7-652a-404b-9527-dbdb81c0306d tempest-ServerDiskConfigTestJSON-1033696558 tempest-ServerDiskConfigTestJSON-1033696558-project-member] Lock "1c1379c2-348b-4be8-9415-594ce53ebf55" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 27.944s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 988.433149] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 988.433149] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Starting heal instance info cache {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9928}} [ 988.433149] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Rebuilding the list of instances to heal {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 988.467071] env[62499]: INFO nova.compute.manager [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 7d91c659-d0ac-435c-af73-d6629bcde6bf] Took 1.03 seconds to deallocate network for instance. [ 988.484550] env[62499]: DEBUG nova.network.neutron [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 988.565523] env[62499]: DEBUG nova.network.neutron [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 988.893842] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Acquiring lock "0cd2c169-ee86-4664-b134-a6c35bde6764" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 988.894167] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Lock "0cd2c169-ee86-4664-b134-a6c35bde6764" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 988.936084] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Didn't find any instances for network info cache update. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10014}} [ 988.936328] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 988.936487] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 988.936635] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 988.936781] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 988.936920] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 988.937078] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 988.937214] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62499) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10547}} [ 988.937631] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager.update_available_resource {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 989.067961] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Releasing lock "refresh_cache-fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 989.068288] env[62499]: DEBUG nova.compute.manager [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 989.068483] env[62499]: DEBUG nova.compute.manager [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 989.068703] env[62499]: DEBUG nova.network.neutron [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 989.083850] env[62499]: DEBUG nova.network.neutron [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 989.397030] env[62499]: DEBUG nova.compute.manager [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 989.440462] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 989.440820] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 989.440881] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 989.441140] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62499) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 989.442097] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd200a44-a933-4537-88c1-f5326365dc99 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 989.451280] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c8cc7e3-f692-41cb-a92d-14c9c8ebdc67 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 989.466972] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-055b8f12-5f3c-496c-ae52-2df5f4668599 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 989.473406] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44a2df12-1cc7-430b-a903-6fbe67592687 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 989.505577] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181514MB free_disk=130GB free_vcpus=48 pci_devices=None {{(pid=62499) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 989.505738] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 989.505936] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 989.508215] env[62499]: INFO nova.scheduler.client.report [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Deleted allocations for instance 7d91c659-d0ac-435c-af73-d6629bcde6bf [ 989.586795] env[62499]: DEBUG nova.network.neutron [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 989.920241] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 990.018070] env[62499]: DEBUG oslo_concurrency.lockutils [None req-5d303a48-fb9a-4ec6-adf5-7102764bb3d0 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "7d91c659-d0ac-435c-af73-d6629bcde6bf" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 28.450s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 990.089599] env[62499]: INFO nova.compute.manager [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab] Took 1.02 seconds to deallocate network for instance. [ 990.750947] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "9d9196c8-3e26-4223-ab13-af4bcd31787b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 990.751278] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "9d9196c8-3e26-4223-ab13-af4bcd31787b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 991.034835] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 991.120241] env[62499]: INFO nova.scheduler.client.report [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Deleted allocations for instance fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab [ 991.253179] env[62499]: DEBUG nova.compute.manager [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 991.537985] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 0cd2c169-ee86-4664-b134-a6c35bde6764 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 991.627987] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b9a61fbf-a336-4e31-afc6-1f3919c80a99 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "fd7ac6e2-5033-4257-b8c4-cff1c56ca8ab" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 19.542s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 991.771220] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 992.040869] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Instance 9d9196c8-3e26-4223-ab13-af4bcd31787b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62499) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 992.041193] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62499) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 992.041421] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62499) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 992.081968] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-513fb7dd-05a8-429a-9dff-ebe52d5ee433 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 992.089936] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4d72273-6c51-46cf-87a7-ef8ac9da5268 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 992.119236] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-933cc970-434f-4987-9c80-02e57c4b0f11 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 992.126303] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ce4542ba-3829-43b1-b604-3ab2eb044ee4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 992.140055] env[62499]: DEBUG nova.compute.provider_tree [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 992.643387] env[62499]: DEBUG nova.scheduler.client.report [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 992.887211] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 992.887506] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 993.148047] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62499) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 993.148362] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 3.642s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 993.148648] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 3.229s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 993.150203] env[62499]: INFO nova.compute.claims [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 993.390059] env[62499]: DEBUG nova.compute.manager [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 993.911792] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 994.203727] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e991de0e-e6bd-453e-8d3f-ab63e4cc44cf {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.211013] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4aa39848-a938-4876-9e86-6c41c413aac4 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.240558] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27bca175-72cb-46d9-9c95-b079b84e0a01 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.246975] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38464aac-5f0b-4e44-969a-8e1c531eafe5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.259435] env[62499]: DEBUG nova.compute.provider_tree [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 994.762564] env[62499]: DEBUG nova.scheduler.client.report [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 995.267724] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.119s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 995.268458] env[62499]: DEBUG nova.compute.manager [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 995.271660] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 3.501s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 995.273122] env[62499]: INFO nova.compute.claims [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 995.778161] env[62499]: DEBUG nova.compute.utils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 995.781373] env[62499]: DEBUG nova.compute.manager [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 995.781546] env[62499]: DEBUG nova.network.neutron [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 995.832406] env[62499]: DEBUG nova.policy [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '48b2a1fbea0e4e22bfa32ec31e438f6d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c21dbc475b0d4225b920fbad495b785a', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 996.080573] env[62499]: DEBUG nova.network.neutron [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Successfully created port: 16bf4b36-3e68-4e34-9943-51e8c0fe805a {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 996.282563] env[62499]: DEBUG nova.compute.manager [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 996.338780] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-13c992fd-0240-43b4-923e-dc5ca44d1e0d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.346580] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7ac1022-1285-4987-ac61-e19eb7eb8bda {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.376039] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-370bb1e7-b7f1-4e8b-95e7-eb9977ff9b1e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.382646] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96a45923-27c6-43b9-8f61-a93126f68f1d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.395033] env[62499]: DEBUG nova.compute.provider_tree [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 996.687129] env[62499]: DEBUG nova.compute.manager [req-6f67bc38-5575-4784-b6c9-3ff447209a6f req-ca7c8009-bb21-4ded-b4bd-0800797d4c49 service nova] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Received event network-changed-16bf4b36-3e68-4e34-9943-51e8c0fe805a {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 996.687129] env[62499]: DEBUG nova.compute.manager [req-6f67bc38-5575-4784-b6c9-3ff447209a6f req-ca7c8009-bb21-4ded-b4bd-0800797d4c49 service nova] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Refreshing instance network info cache due to event network-changed-16bf4b36-3e68-4e34-9943-51e8c0fe805a. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 996.687129] env[62499]: DEBUG oslo_concurrency.lockutils [req-6f67bc38-5575-4784-b6c9-3ff447209a6f req-ca7c8009-bb21-4ded-b4bd-0800797d4c49 service nova] Acquiring lock "refresh_cache-0cd2c169-ee86-4664-b134-a6c35bde6764" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 996.687129] env[62499]: DEBUG oslo_concurrency.lockutils [req-6f67bc38-5575-4784-b6c9-3ff447209a6f req-ca7c8009-bb21-4ded-b4bd-0800797d4c49 service nova] Acquired lock "refresh_cache-0cd2c169-ee86-4664-b134-a6c35bde6764" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 996.687462] env[62499]: DEBUG nova.network.neutron [req-6f67bc38-5575-4784-b6c9-3ff447209a6f req-ca7c8009-bb21-4ded-b4bd-0800797d4c49 service nova] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Refreshing network info cache for port 16bf4b36-3e68-4e34-9943-51e8c0fe805a {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 996.848936] env[62499]: ERROR nova.compute.manager [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 16bf4b36-3e68-4e34-9943-51e8c0fe805a, please check neutron logs for more information. [ 996.848936] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 996.848936] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 996.848936] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 996.848936] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 996.848936] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 996.848936] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 996.848936] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 996.848936] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 996.848936] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 996.848936] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 996.848936] env[62499]: ERROR nova.compute.manager raise self.value [ 996.848936] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 996.848936] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 996.848936] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 996.848936] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 996.849445] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 996.849445] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 996.849445] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 16bf4b36-3e68-4e34-9943-51e8c0fe805a, please check neutron logs for more information. [ 996.849445] env[62499]: ERROR nova.compute.manager [ 996.849445] env[62499]: Traceback (most recent call last): [ 996.849445] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 996.849445] env[62499]: listener.cb(fileno) [ 996.849445] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 996.849445] env[62499]: result = function(*args, **kwargs) [ 996.849445] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 996.849445] env[62499]: return func(*args, **kwargs) [ 996.849445] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 996.849445] env[62499]: raise e [ 996.849445] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 996.849445] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 996.849445] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 996.849445] env[62499]: created_port_ids = self._update_ports_for_instance( [ 996.849445] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 996.849445] env[62499]: with excutils.save_and_reraise_exception(): [ 996.849445] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 996.849445] env[62499]: self.force_reraise() [ 996.849445] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 996.849445] env[62499]: raise self.value [ 996.849445] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 996.849445] env[62499]: updated_port = self._update_port( [ 996.849445] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 996.849445] env[62499]: _ensure_no_port_binding_failure(port) [ 996.849445] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 996.849445] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 996.850454] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 16bf4b36-3e68-4e34-9943-51e8c0fe805a, please check neutron logs for more information. [ 996.850454] env[62499]: Removing descriptor: 16 [ 996.898495] env[62499]: DEBUG nova.scheduler.client.report [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 997.205828] env[62499]: DEBUG nova.network.neutron [req-6f67bc38-5575-4784-b6c9-3ff447209a6f req-ca7c8009-bb21-4ded-b4bd-0800797d4c49 service nova] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 997.280589] env[62499]: DEBUG nova.network.neutron [req-6f67bc38-5575-4784-b6c9-3ff447209a6f req-ca7c8009-bb21-4ded-b4bd-0800797d4c49 service nova] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 997.294667] env[62499]: DEBUG nova.compute.manager [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 997.318681] env[62499]: DEBUG nova.virt.hardware [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 997.319050] env[62499]: DEBUG nova.virt.hardware [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 997.319176] env[62499]: DEBUG nova.virt.hardware [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 997.319396] env[62499]: DEBUG nova.virt.hardware [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 997.319564] env[62499]: DEBUG nova.virt.hardware [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 997.319712] env[62499]: DEBUG nova.virt.hardware [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 997.319917] env[62499]: DEBUG nova.virt.hardware [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 997.320168] env[62499]: DEBUG nova.virt.hardware [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 997.320363] env[62499]: DEBUG nova.virt.hardware [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 997.320528] env[62499]: DEBUG nova.virt.hardware [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 997.320698] env[62499]: DEBUG nova.virt.hardware [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 997.321534] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4c10549b-3679-4730-95c1-f5c4bf72dd18 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 997.329360] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-812e5b86-3f49-46ea-b4dd-0741ea1e4708 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 997.342611] env[62499]: ERROR nova.compute.manager [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 16bf4b36-3e68-4e34-9943-51e8c0fe805a, please check neutron logs for more information. [ 997.342611] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Traceback (most recent call last): [ 997.342611] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 997.342611] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] yield resources [ 997.342611] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 997.342611] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] self.driver.spawn(context, instance, image_meta, [ 997.342611] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 997.342611] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] self._vmops.spawn(context, instance, image_meta, injected_files, [ 997.342611] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 997.342611] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] vm_ref = self.build_virtual_machine(instance, [ 997.342611] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 997.343034] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] vif_infos = vmwarevif.get_vif_info(self._session, [ 997.343034] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 997.343034] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] for vif in network_info: [ 997.343034] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 997.343034] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] return self._sync_wrapper(fn, *args, **kwargs) [ 997.343034] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 997.343034] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] self.wait() [ 997.343034] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 997.343034] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] self[:] = self._gt.wait() [ 997.343034] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 997.343034] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] return self._exit_event.wait() [ 997.343034] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 997.343034] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] current.throw(*self._exc) [ 997.343436] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 997.343436] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] result = function(*args, **kwargs) [ 997.343436] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 997.343436] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] return func(*args, **kwargs) [ 997.343436] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 997.343436] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] raise e [ 997.343436] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 997.343436] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] nwinfo = self.network_api.allocate_for_instance( [ 997.343436] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 997.343436] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] created_port_ids = self._update_ports_for_instance( [ 997.343436] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 997.343436] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] with excutils.save_and_reraise_exception(): [ 997.343436] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 997.343837] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] self.force_reraise() [ 997.343837] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 997.343837] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] raise self.value [ 997.343837] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 997.343837] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] updated_port = self._update_port( [ 997.343837] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 997.343837] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] _ensure_no_port_binding_failure(port) [ 997.343837] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 997.343837] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] raise exception.PortBindingFailed(port_id=port['id']) [ 997.343837] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] nova.exception.PortBindingFailed: Binding failed for port 16bf4b36-3e68-4e34-9943-51e8c0fe805a, please check neutron logs for more information. [ 997.343837] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] [ 997.343837] env[62499]: INFO nova.compute.manager [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Terminating instance [ 997.344807] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Acquiring lock "refresh_cache-0cd2c169-ee86-4664-b134-a6c35bde6764" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 997.403116] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.131s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 997.403636] env[62499]: DEBUG nova.compute.manager [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 997.406520] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 3.495s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 997.408376] env[62499]: INFO nova.compute.claims [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 997.783345] env[62499]: DEBUG oslo_concurrency.lockutils [req-6f67bc38-5575-4784-b6c9-3ff447209a6f req-ca7c8009-bb21-4ded-b4bd-0800797d4c49 service nova] Releasing lock "refresh_cache-0cd2c169-ee86-4664-b134-a6c35bde6764" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 997.783730] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Acquired lock "refresh_cache-0cd2c169-ee86-4664-b134-a6c35bde6764" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 997.783925] env[62499]: DEBUG nova.network.neutron [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 997.912308] env[62499]: DEBUG nova.compute.utils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 997.915402] env[62499]: DEBUG nova.compute.manager [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 997.915564] env[62499]: DEBUG nova.network.neutron [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 997.952610] env[62499]: DEBUG nova.policy [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7da3078e8bb5444c96e007cfc2d57952', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c9f95d7e954f4413b01aad519c498967', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 998.242115] env[62499]: DEBUG nova.network.neutron [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Successfully created port: 846f3831-1fe9-4a19-96b3-87096dd5a2d4 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 998.302334] env[62499]: DEBUG nova.network.neutron [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 998.373637] env[62499]: DEBUG nova.network.neutron [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 998.420713] env[62499]: DEBUG nova.compute.manager [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 998.552682] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6119844-bc7d-43e2-a3a2-62e5f6ce550a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.560429] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-57c45b3f-fbe8-4487-9df3-ca61f5681c28 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.589711] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a543446-c86b-431b-a85a-2358a2e57ec6 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.597037] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1fd6fce7-8d9a-4824-8d71-df15972df1eb {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.611924] env[62499]: DEBUG nova.compute.provider_tree [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 998.712674] env[62499]: DEBUG nova.compute.manager [req-dbfe5e38-2e85-48c9-a931-9191a51a96c2 req-d2ce866b-4825-4c6f-8c60-84dc8a0f948d service nova] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Received event network-vif-deleted-16bf4b36-3e68-4e34-9943-51e8c0fe805a {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 998.876800] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Releasing lock "refresh_cache-0cd2c169-ee86-4664-b134-a6c35bde6764" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 998.877301] env[62499]: DEBUG nova.compute.manager [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 998.877439] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 998.878081] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c79a5046-ee47-4be0-8653-609620eabba8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.887248] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f48df737-3b48-49e3-a1b6-78928da8848e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.908999] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 0cd2c169-ee86-4664-b134-a6c35bde6764 could not be found. [ 998.909245] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 998.909427] env[62499]: INFO nova.compute.manager [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Took 0.03 seconds to destroy the instance on the hypervisor. [ 998.909687] env[62499]: DEBUG oslo.service.loopingcall [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 998.909869] env[62499]: DEBUG nova.compute.manager [-] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 998.909961] env[62499]: DEBUG nova.network.neutron [-] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 998.927542] env[62499]: DEBUG nova.network.neutron [-] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 999.032095] env[62499]: ERROR nova.compute.manager [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 846f3831-1fe9-4a19-96b3-87096dd5a2d4, please check neutron logs for more information. [ 999.032095] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 999.032095] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 999.032095] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 999.032095] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 999.032095] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 999.032095] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 999.032095] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 999.032095] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 999.032095] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 999.032095] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 999.032095] env[62499]: ERROR nova.compute.manager raise self.value [ 999.032095] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 999.032095] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 999.032095] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 999.032095] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 999.032905] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 999.032905] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 999.032905] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 846f3831-1fe9-4a19-96b3-87096dd5a2d4, please check neutron logs for more information. [ 999.032905] env[62499]: ERROR nova.compute.manager [ 999.032905] env[62499]: Traceback (most recent call last): [ 999.032905] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 999.032905] env[62499]: listener.cb(fileno) [ 999.032905] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 999.032905] env[62499]: result = function(*args, **kwargs) [ 999.032905] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 999.032905] env[62499]: return func(*args, **kwargs) [ 999.032905] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 999.032905] env[62499]: raise e [ 999.032905] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 999.032905] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 999.032905] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 999.032905] env[62499]: created_port_ids = self._update_ports_for_instance( [ 999.032905] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 999.032905] env[62499]: with excutils.save_and_reraise_exception(): [ 999.032905] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 999.032905] env[62499]: self.force_reraise() [ 999.032905] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 999.032905] env[62499]: raise self.value [ 999.032905] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 999.032905] env[62499]: updated_port = self._update_port( [ 999.032905] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 999.032905] env[62499]: _ensure_no_port_binding_failure(port) [ 999.032905] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 999.032905] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 999.034329] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 846f3831-1fe9-4a19-96b3-87096dd5a2d4, please check neutron logs for more information. [ 999.034329] env[62499]: Removing descriptor: 16 [ 999.113135] env[62499]: DEBUG nova.scheduler.client.report [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 999.430443] env[62499]: DEBUG nova.compute.manager [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 999.432493] env[62499]: DEBUG nova.network.neutron [-] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 999.455839] env[62499]: DEBUG nova.virt.hardware [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 999.456094] env[62499]: DEBUG nova.virt.hardware [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 999.456258] env[62499]: DEBUG nova.virt.hardware [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 999.456437] env[62499]: DEBUG nova.virt.hardware [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 999.456582] env[62499]: DEBUG nova.virt.hardware [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 999.456727] env[62499]: DEBUG nova.virt.hardware [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 999.456930] env[62499]: DEBUG nova.virt.hardware [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 999.457104] env[62499]: DEBUG nova.virt.hardware [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 999.457275] env[62499]: DEBUG nova.virt.hardware [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 999.457434] env[62499]: DEBUG nova.virt.hardware [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 999.457604] env[62499]: DEBUG nova.virt.hardware [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 999.458473] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2bbf9009-22ca-406b-b74d-7b9c5606d895 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.466720] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-41c82154-85cc-452f-ab99-4db949c0a6f5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.480430] env[62499]: ERROR nova.compute.manager [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 846f3831-1fe9-4a19-96b3-87096dd5a2d4, please check neutron logs for more information. [ 999.480430] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Traceback (most recent call last): [ 999.480430] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 999.480430] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] yield resources [ 999.480430] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 999.480430] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] self.driver.spawn(context, instance, image_meta, [ 999.480430] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 999.480430] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 999.480430] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 999.480430] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] vm_ref = self.build_virtual_machine(instance, [ 999.480430] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 999.480920] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] vif_infos = vmwarevif.get_vif_info(self._session, [ 999.480920] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 999.480920] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] for vif in network_info: [ 999.480920] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 999.480920] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] return self._sync_wrapper(fn, *args, **kwargs) [ 999.480920] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 999.480920] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] self.wait() [ 999.480920] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 999.480920] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] self[:] = self._gt.wait() [ 999.480920] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 999.480920] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] return self._exit_event.wait() [ 999.480920] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 999.480920] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] current.throw(*self._exc) [ 999.481470] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 999.481470] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] result = function(*args, **kwargs) [ 999.481470] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 999.481470] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] return func(*args, **kwargs) [ 999.481470] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 999.481470] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] raise e [ 999.481470] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 999.481470] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] nwinfo = self.network_api.allocate_for_instance( [ 999.481470] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 999.481470] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] created_port_ids = self._update_ports_for_instance( [ 999.481470] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 999.481470] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] with excutils.save_and_reraise_exception(): [ 999.481470] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 999.481940] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] self.force_reraise() [ 999.481940] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 999.481940] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] raise self.value [ 999.481940] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 999.481940] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] updated_port = self._update_port( [ 999.481940] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 999.481940] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] _ensure_no_port_binding_failure(port) [ 999.481940] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 999.481940] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] raise exception.PortBindingFailed(port_id=port['id']) [ 999.481940] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] nova.exception.PortBindingFailed: Binding failed for port 846f3831-1fe9-4a19-96b3-87096dd5a2d4, please check neutron logs for more information. [ 999.481940] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] [ 999.481940] env[62499]: INFO nova.compute.manager [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Terminating instance [ 999.482759] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "refresh_cache-9d9196c8-3e26-4223-ab13-af4bcd31787b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 999.482915] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquired lock "refresh_cache-9d9196c8-3e26-4223-ab13-af4bcd31787b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 999.483091] env[62499]: DEBUG nova.network.neutron [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 999.617532] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.211s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 999.618207] env[62499]: DEBUG nova.compute.manager [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 999.934946] env[62499]: INFO nova.compute.manager [-] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Took 1.02 seconds to deallocate network for instance. [ 999.937218] env[62499]: DEBUG nova.compute.claims [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 999.937397] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 999.937611] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1000.000585] env[62499]: DEBUG nova.network.neutron [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1000.088854] env[62499]: DEBUG nova.network.neutron [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1000.123224] env[62499]: DEBUG nova.compute.utils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1000.124836] env[62499]: DEBUG nova.compute.manager [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1000.125013] env[62499]: DEBUG nova.network.neutron [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1000.161692] env[62499]: DEBUG nova.policy [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '02640299eb9c4a65bc80de6242dd2cd5', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f509dfe537d441bcacb1a4aab3607214', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 1000.384518] env[62499]: DEBUG nova.network.neutron [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Successfully created port: 334799b5-89ef-45c5-b835-f2b50f55d337 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1000.495027] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-71fceb85-44f0-43de-bfc7-28a4ea0ffc97 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.502370] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb497042-f424-4d27-80b6-2b29aa9d3960 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.532225] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e80b978-9475-46d7-8db1-0c7b466ee5ca {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.539318] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-278715c6-462e-4d50-a760-7d13c145ff62 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.552082] env[62499]: DEBUG nova.compute.provider_tree [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1000.591980] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Releasing lock "refresh_cache-9d9196c8-3e26-4223-ab13-af4bcd31787b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1000.592426] env[62499]: DEBUG nova.compute.manager [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1000.592630] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1000.592901] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c7e55c67-5e88-4917-abbf-77cb139e960d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.601319] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4ef413c-8f53-4e9f-a158-f9c2d0eaa3ff {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.621401] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9d9196c8-3e26-4223-ab13-af4bcd31787b could not be found. [ 1000.621602] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1000.621780] env[62499]: INFO nova.compute.manager [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1000.622239] env[62499]: DEBUG oslo.service.loopingcall [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1000.622239] env[62499]: DEBUG nova.compute.manager [-] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1000.622361] env[62499]: DEBUG nova.network.neutron [-] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1000.628862] env[62499]: DEBUG nova.compute.manager [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1000.637554] env[62499]: DEBUG nova.network.neutron [-] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1000.742841] env[62499]: DEBUG nova.compute.manager [req-e50c6268-47bc-4615-8f83-70173028e4e2 req-8bdff698-4fbf-4261-aea2-5d74f7eb9929 service nova] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Received event network-changed-846f3831-1fe9-4a19-96b3-87096dd5a2d4 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 1000.743370] env[62499]: DEBUG nova.compute.manager [req-e50c6268-47bc-4615-8f83-70173028e4e2 req-8bdff698-4fbf-4261-aea2-5d74f7eb9929 service nova] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Refreshing instance network info cache due to event network-changed-846f3831-1fe9-4a19-96b3-87096dd5a2d4. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 1000.743370] env[62499]: DEBUG oslo_concurrency.lockutils [req-e50c6268-47bc-4615-8f83-70173028e4e2 req-8bdff698-4fbf-4261-aea2-5d74f7eb9929 service nova] Acquiring lock "refresh_cache-9d9196c8-3e26-4223-ab13-af4bcd31787b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1000.743500] env[62499]: DEBUG oslo_concurrency.lockutils [req-e50c6268-47bc-4615-8f83-70173028e4e2 req-8bdff698-4fbf-4261-aea2-5d74f7eb9929 service nova] Acquired lock "refresh_cache-9d9196c8-3e26-4223-ab13-af4bcd31787b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1000.743542] env[62499]: DEBUG nova.network.neutron [req-e50c6268-47bc-4615-8f83-70173028e4e2 req-8bdff698-4fbf-4261-aea2-5d74f7eb9929 service nova] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Refreshing network info cache for port 846f3831-1fe9-4a19-96b3-87096dd5a2d4 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1001.055252] env[62499]: DEBUG nova.scheduler.client.report [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1001.139321] env[62499]: DEBUG nova.network.neutron [-] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1001.261782] env[62499]: DEBUG nova.network.neutron [req-e50c6268-47bc-4615-8f83-70173028e4e2 req-8bdff698-4fbf-4261-aea2-5d74f7eb9929 service nova] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1001.331751] env[62499]: ERROR nova.compute.manager [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 334799b5-89ef-45c5-b835-f2b50f55d337, please check neutron logs for more information. [ 1001.331751] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 1001.331751] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1001.331751] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1001.331751] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1001.331751] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1001.331751] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1001.331751] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1001.331751] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1001.331751] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 1001.331751] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1001.331751] env[62499]: ERROR nova.compute.manager raise self.value [ 1001.331751] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1001.331751] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 1001.331751] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1001.331751] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1001.332687] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1001.332687] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1001.332687] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 334799b5-89ef-45c5-b835-f2b50f55d337, please check neutron logs for more information. [ 1001.332687] env[62499]: ERROR nova.compute.manager [ 1001.332687] env[62499]: Traceback (most recent call last): [ 1001.332687] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1001.332687] env[62499]: listener.cb(fileno) [ 1001.332687] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1001.332687] env[62499]: result = function(*args, **kwargs) [ 1001.332687] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1001.332687] env[62499]: return func(*args, **kwargs) [ 1001.332687] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1001.332687] env[62499]: raise e [ 1001.332687] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1001.332687] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 1001.332687] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1001.332687] env[62499]: created_port_ids = self._update_ports_for_instance( [ 1001.332687] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1001.332687] env[62499]: with excutils.save_and_reraise_exception(): [ 1001.332687] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1001.332687] env[62499]: self.force_reraise() [ 1001.332687] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1001.332687] env[62499]: raise self.value [ 1001.332687] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1001.332687] env[62499]: updated_port = self._update_port( [ 1001.332687] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1001.332687] env[62499]: _ensure_no_port_binding_failure(port) [ 1001.332687] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1001.332687] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 1001.333824] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 334799b5-89ef-45c5-b835-f2b50f55d337, please check neutron logs for more information. [ 1001.333824] env[62499]: Removing descriptor: 16 [ 1001.337235] env[62499]: DEBUG nova.network.neutron [req-e50c6268-47bc-4615-8f83-70173028e4e2 req-8bdff698-4fbf-4261-aea2-5d74f7eb9929 service nova] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1001.559552] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.622s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1001.560224] env[62499]: ERROR nova.compute.manager [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 16bf4b36-3e68-4e34-9943-51e8c0fe805a, please check neutron logs for more information. [ 1001.560224] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Traceback (most recent call last): [ 1001.560224] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1001.560224] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] self.driver.spawn(context, instance, image_meta, [ 1001.560224] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 1001.560224] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1001.560224] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1001.560224] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] vm_ref = self.build_virtual_machine(instance, [ 1001.560224] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1001.560224] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] vif_infos = vmwarevif.get_vif_info(self._session, [ 1001.560224] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1001.560650] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] for vif in network_info: [ 1001.560650] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1001.560650] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] return self._sync_wrapper(fn, *args, **kwargs) [ 1001.560650] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1001.560650] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] self.wait() [ 1001.560650] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1001.560650] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] self[:] = self._gt.wait() [ 1001.560650] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1001.560650] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] return self._exit_event.wait() [ 1001.560650] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1001.560650] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] current.throw(*self._exc) [ 1001.560650] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1001.560650] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] result = function(*args, **kwargs) [ 1001.561036] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1001.561036] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] return func(*args, **kwargs) [ 1001.561036] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1001.561036] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] raise e [ 1001.561036] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1001.561036] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] nwinfo = self.network_api.allocate_for_instance( [ 1001.561036] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1001.561036] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] created_port_ids = self._update_ports_for_instance( [ 1001.561036] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1001.561036] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] with excutils.save_and_reraise_exception(): [ 1001.561036] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1001.561036] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] self.force_reraise() [ 1001.561036] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1001.561441] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] raise self.value [ 1001.561441] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1001.561441] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] updated_port = self._update_port( [ 1001.561441] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1001.561441] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] _ensure_no_port_binding_failure(port) [ 1001.561441] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1001.561441] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] raise exception.PortBindingFailed(port_id=port['id']) [ 1001.561441] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] nova.exception.PortBindingFailed: Binding failed for port 16bf4b36-3e68-4e34-9943-51e8c0fe805a, please check neutron logs for more information. [ 1001.561441] env[62499]: ERROR nova.compute.manager [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] [ 1001.561441] env[62499]: DEBUG nova.compute.utils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Binding failed for port 16bf4b36-3e68-4e34-9943-51e8c0fe805a, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1001.562959] env[62499]: DEBUG nova.compute.manager [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Build of instance 0cd2c169-ee86-4664-b134-a6c35bde6764 was re-scheduled: Binding failed for port 16bf4b36-3e68-4e34-9943-51e8c0fe805a, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1001.563373] env[62499]: DEBUG nova.compute.manager [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1001.563600] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Acquiring lock "refresh_cache-0cd2c169-ee86-4664-b134-a6c35bde6764" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1001.563743] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Acquired lock "refresh_cache-0cd2c169-ee86-4664-b134-a6c35bde6764" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1001.563900] env[62499]: DEBUG nova.network.neutron [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1001.638196] env[62499]: DEBUG nova.compute.manager [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1001.641943] env[62499]: INFO nova.compute.manager [-] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Took 1.02 seconds to deallocate network for instance. [ 1001.644009] env[62499]: DEBUG nova.compute.claims [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1001.644195] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1001.644409] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1001.662840] env[62499]: DEBUG nova.virt.hardware [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1001.663156] env[62499]: DEBUG nova.virt.hardware [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1001.663386] env[62499]: DEBUG nova.virt.hardware [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1001.663650] env[62499]: DEBUG nova.virt.hardware [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1001.663867] env[62499]: DEBUG nova.virt.hardware [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1001.664105] env[62499]: DEBUG nova.virt.hardware [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1001.664386] env[62499]: DEBUG nova.virt.hardware [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1001.664614] env[62499]: DEBUG nova.virt.hardware [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1001.664854] env[62499]: DEBUG nova.virt.hardware [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1001.665114] env[62499]: DEBUG nova.virt.hardware [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1001.665371] env[62499]: DEBUG nova.virt.hardware [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1001.666263] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2c4a3a1-887b-42ce-9359-3f32d4cc3890 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1001.674577] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4a9a532-0c08-44da-9046-6de4bb4ccbd5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1001.688790] env[62499]: ERROR nova.compute.manager [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 334799b5-89ef-45c5-b835-f2b50f55d337, please check neutron logs for more information. [ 1001.688790] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Traceback (most recent call last): [ 1001.688790] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1001.688790] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] yield resources [ 1001.688790] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1001.688790] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] self.driver.spawn(context, instance, image_meta, [ 1001.688790] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 1001.688790] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1001.688790] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1001.688790] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] vm_ref = self.build_virtual_machine(instance, [ 1001.688790] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1001.689563] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] vif_infos = vmwarevif.get_vif_info(self._session, [ 1001.689563] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1001.689563] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] for vif in network_info: [ 1001.689563] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1001.689563] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] return self._sync_wrapper(fn, *args, **kwargs) [ 1001.689563] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1001.689563] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] self.wait() [ 1001.689563] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1001.689563] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] self[:] = self._gt.wait() [ 1001.689563] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1001.689563] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] return self._exit_event.wait() [ 1001.689563] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1001.689563] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] current.throw(*self._exc) [ 1001.690355] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1001.690355] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] result = function(*args, **kwargs) [ 1001.690355] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1001.690355] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] return func(*args, **kwargs) [ 1001.690355] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1001.690355] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] raise e [ 1001.690355] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1001.690355] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] nwinfo = self.network_api.allocate_for_instance( [ 1001.690355] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1001.690355] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] created_port_ids = self._update_ports_for_instance( [ 1001.690355] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1001.690355] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] with excutils.save_and_reraise_exception(): [ 1001.690355] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1001.691141] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] self.force_reraise() [ 1001.691141] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1001.691141] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] raise self.value [ 1001.691141] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1001.691141] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] updated_port = self._update_port( [ 1001.691141] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1001.691141] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] _ensure_no_port_binding_failure(port) [ 1001.691141] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1001.691141] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] raise exception.PortBindingFailed(port_id=port['id']) [ 1001.691141] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] nova.exception.PortBindingFailed: Binding failed for port 334799b5-89ef-45c5-b835-f2b50f55d337, please check neutron logs for more information. [ 1001.691141] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] [ 1001.691141] env[62499]: INFO nova.compute.manager [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Terminating instance [ 1001.691886] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "refresh_cache-3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1001.691965] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquired lock "refresh_cache-3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1001.692219] env[62499]: DEBUG nova.network.neutron [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1001.840094] env[62499]: DEBUG oslo_concurrency.lockutils [req-e50c6268-47bc-4615-8f83-70173028e4e2 req-8bdff698-4fbf-4261-aea2-5d74f7eb9929 service nova] Releasing lock "refresh_cache-9d9196c8-3e26-4223-ab13-af4bcd31787b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1001.840389] env[62499]: DEBUG nova.compute.manager [req-e50c6268-47bc-4615-8f83-70173028e4e2 req-8bdff698-4fbf-4261-aea2-5d74f7eb9929 service nova] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Received event network-vif-deleted-846f3831-1fe9-4a19-96b3-87096dd5a2d4 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 1002.081255] env[62499]: DEBUG nova.network.neutron [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1002.161578] env[62499]: DEBUG nova.network.neutron [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1002.199853] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20854ab6-76c4-44de-9a9b-fd7274d73dd7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1002.207566] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8b36c8b-80aa-4542-85f2-beb31dbee39b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1002.210850] env[62499]: DEBUG nova.network.neutron [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1002.239725] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b53a6517-dea9-4569-83ec-b8f58be38c6c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1002.248415] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e8ccd64-16bf-4c47-acf8-a3e9a2413037 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1002.261371] env[62499]: DEBUG nova.compute.provider_tree [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1002.283900] env[62499]: DEBUG nova.network.neutron [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1002.664458] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Releasing lock "refresh_cache-0cd2c169-ee86-4664-b134-a6c35bde6764" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1002.664895] env[62499]: DEBUG nova.compute.manager [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1002.664895] env[62499]: DEBUG nova.compute.manager [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1002.665056] env[62499]: DEBUG nova.network.neutron [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1002.680617] env[62499]: DEBUG nova.network.neutron [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1002.764763] env[62499]: DEBUG nova.scheduler.client.report [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1002.770603] env[62499]: DEBUG nova.compute.manager [req-bdcef082-d866-42c7-93e1-a21c5c5ff0bb req-9e14aeb3-31b1-4023-a04a-2435453548b9 service nova] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Received event network-changed-334799b5-89ef-45c5-b835-f2b50f55d337 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 1002.770779] env[62499]: DEBUG nova.compute.manager [req-bdcef082-d866-42c7-93e1-a21c5c5ff0bb req-9e14aeb3-31b1-4023-a04a-2435453548b9 service nova] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Refreshing instance network info cache due to event network-changed-334799b5-89ef-45c5-b835-f2b50f55d337. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 1002.770966] env[62499]: DEBUG oslo_concurrency.lockutils [req-bdcef082-d866-42c7-93e1-a21c5c5ff0bb req-9e14aeb3-31b1-4023-a04a-2435453548b9 service nova] Acquiring lock "refresh_cache-3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1002.786171] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Releasing lock "refresh_cache-3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1002.786578] env[62499]: DEBUG nova.compute.manager [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1002.786767] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1002.787294] env[62499]: DEBUG oslo_concurrency.lockutils [req-bdcef082-d866-42c7-93e1-a21c5c5ff0bb req-9e14aeb3-31b1-4023-a04a-2435453548b9 service nova] Acquired lock "refresh_cache-3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1002.787471] env[62499]: DEBUG nova.network.neutron [req-bdcef082-d866-42c7-93e1-a21c5c5ff0bb req-9e14aeb3-31b1-4023-a04a-2435453548b9 service nova] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Refreshing network info cache for port 334799b5-89ef-45c5-b835-f2b50f55d337 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1002.788383] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-181e9dd8-3027-4ea6-ad71-e3230040d111 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1002.797837] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a7142fc-2fdf-4848-9c1e-0a24ff96533f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1002.818884] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2 could not be found. [ 1002.819101] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1002.819343] env[62499]: INFO nova.compute.manager [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1002.819593] env[62499]: DEBUG oslo.service.loopingcall [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1002.819798] env[62499]: DEBUG nova.compute.manager [-] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1002.819890] env[62499]: DEBUG nova.network.neutron [-] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1002.834640] env[62499]: DEBUG nova.network.neutron [-] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1003.183212] env[62499]: DEBUG nova.network.neutron [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1003.270828] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.626s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1003.271486] env[62499]: ERROR nova.compute.manager [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 846f3831-1fe9-4a19-96b3-87096dd5a2d4, please check neutron logs for more information. [ 1003.271486] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Traceback (most recent call last): [ 1003.271486] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1003.271486] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] self.driver.spawn(context, instance, image_meta, [ 1003.271486] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 1003.271486] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1003.271486] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1003.271486] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] vm_ref = self.build_virtual_machine(instance, [ 1003.271486] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1003.271486] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] vif_infos = vmwarevif.get_vif_info(self._session, [ 1003.271486] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1003.271885] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] for vif in network_info: [ 1003.271885] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1003.271885] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] return self._sync_wrapper(fn, *args, **kwargs) [ 1003.271885] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1003.271885] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] self.wait() [ 1003.271885] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1003.271885] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] self[:] = self._gt.wait() [ 1003.271885] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1003.271885] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] return self._exit_event.wait() [ 1003.271885] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1003.271885] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] current.throw(*self._exc) [ 1003.271885] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1003.271885] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] result = function(*args, **kwargs) [ 1003.272415] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1003.272415] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] return func(*args, **kwargs) [ 1003.272415] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1003.272415] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] raise e [ 1003.272415] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1003.272415] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] nwinfo = self.network_api.allocate_for_instance( [ 1003.272415] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1003.272415] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] created_port_ids = self._update_ports_for_instance( [ 1003.272415] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1003.272415] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] with excutils.save_and_reraise_exception(): [ 1003.272415] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1003.272415] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] self.force_reraise() [ 1003.272415] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1003.272852] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] raise self.value [ 1003.272852] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1003.272852] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] updated_port = self._update_port( [ 1003.272852] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1003.272852] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] _ensure_no_port_binding_failure(port) [ 1003.272852] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1003.272852] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] raise exception.PortBindingFailed(port_id=port['id']) [ 1003.272852] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] nova.exception.PortBindingFailed: Binding failed for port 846f3831-1fe9-4a19-96b3-87096dd5a2d4, please check neutron logs for more information. [ 1003.272852] env[62499]: ERROR nova.compute.manager [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] [ 1003.272852] env[62499]: DEBUG nova.compute.utils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Binding failed for port 846f3831-1fe9-4a19-96b3-87096dd5a2d4, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1003.273801] env[62499]: DEBUG nova.compute.manager [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Build of instance 9d9196c8-3e26-4223-ab13-af4bcd31787b was re-scheduled: Binding failed for port 846f3831-1fe9-4a19-96b3-87096dd5a2d4, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1003.274214] env[62499]: DEBUG nova.compute.manager [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1003.274441] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "refresh_cache-9d9196c8-3e26-4223-ab13-af4bcd31787b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1003.274586] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquired lock "refresh_cache-9d9196c8-3e26-4223-ab13-af4bcd31787b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1003.274744] env[62499]: DEBUG nova.network.neutron [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1003.305796] env[62499]: DEBUG nova.network.neutron [req-bdcef082-d866-42c7-93e1-a21c5c5ff0bb req-9e14aeb3-31b1-4023-a04a-2435453548b9 service nova] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1003.336362] env[62499]: DEBUG nova.network.neutron [-] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1003.370918] env[62499]: DEBUG nova.network.neutron [req-bdcef082-d866-42c7-93e1-a21c5c5ff0bb req-9e14aeb3-31b1-4023-a04a-2435453548b9 service nova] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1003.685669] env[62499]: INFO nova.compute.manager [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] [instance: 0cd2c169-ee86-4664-b134-a6c35bde6764] Took 1.02 seconds to deallocate network for instance. [ 1003.793137] env[62499]: DEBUG nova.network.neutron [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1003.837962] env[62499]: INFO nova.compute.manager [-] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Took 1.02 seconds to deallocate network for instance. [ 1003.840120] env[62499]: DEBUG nova.compute.claims [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1003.840299] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1003.840505] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1003.859763] env[62499]: DEBUG nova.network.neutron [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1003.872738] env[62499]: DEBUG oslo_concurrency.lockutils [req-bdcef082-d866-42c7-93e1-a21c5c5ff0bb req-9e14aeb3-31b1-4023-a04a-2435453548b9 service nova] Releasing lock "refresh_cache-3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1003.872967] env[62499]: DEBUG nova.compute.manager [req-bdcef082-d866-42c7-93e1-a21c5c5ff0bb req-9e14aeb3-31b1-4023-a04a-2435453548b9 service nova] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Received event network-vif-deleted-334799b5-89ef-45c5-b835-f2b50f55d337 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 1004.362752] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Releasing lock "refresh_cache-9d9196c8-3e26-4223-ab13-af4bcd31787b" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1004.362981] env[62499]: DEBUG nova.compute.manager [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1004.363184] env[62499]: DEBUG nova.compute.manager [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1004.363355] env[62499]: DEBUG nova.network.neutron [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1004.378350] env[62499]: DEBUG nova.network.neutron [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1004.396577] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-034c448f-ffac-47d8-b5e8-1dcf8fdb34a0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1004.404396] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3c98a197-1df5-4b31-800c-07bc4071d12a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1004.432952] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d7f7cc0-57d9-4186-9015-5662532f6971 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1004.439331] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be9257ba-feb6-422c-b32b-74e216effb64 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1004.452687] env[62499]: DEBUG nova.compute.provider_tree [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1004.711051] env[62499]: INFO nova.scheduler.client.report [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Deleted allocations for instance 0cd2c169-ee86-4664-b134-a6c35bde6764 [ 1004.880689] env[62499]: DEBUG nova.network.neutron [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1004.955620] env[62499]: DEBUG nova.scheduler.client.report [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1005.219051] env[62499]: DEBUG oslo_concurrency.lockutils [None req-66aef1cf-2289-4e5d-ae5b-3280a8f721c8 tempest-AttachVolumeShelveTestJSON-2006650516 tempest-AttachVolumeShelveTestJSON-2006650516-project-member] Lock "0cd2c169-ee86-4664-b134-a6c35bde6764" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 16.325s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1005.385405] env[62499]: INFO nova.compute.manager [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 9d9196c8-3e26-4223-ab13-af4bcd31787b] Took 1.02 seconds to deallocate network for instance. [ 1005.460416] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.620s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1005.461047] env[62499]: ERROR nova.compute.manager [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 334799b5-89ef-45c5-b835-f2b50f55d337, please check neutron logs for more information. [ 1005.461047] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Traceback (most recent call last): [ 1005.461047] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1005.461047] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] self.driver.spawn(context, instance, image_meta, [ 1005.461047] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 1005.461047] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1005.461047] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1005.461047] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] vm_ref = self.build_virtual_machine(instance, [ 1005.461047] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1005.461047] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] vif_infos = vmwarevif.get_vif_info(self._session, [ 1005.461047] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1005.461536] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] for vif in network_info: [ 1005.461536] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1005.461536] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] return self._sync_wrapper(fn, *args, **kwargs) [ 1005.461536] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1005.461536] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] self.wait() [ 1005.461536] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1005.461536] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] self[:] = self._gt.wait() [ 1005.461536] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1005.461536] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] return self._exit_event.wait() [ 1005.461536] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1005.461536] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] current.throw(*self._exc) [ 1005.461536] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1005.461536] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] result = function(*args, **kwargs) [ 1005.461921] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1005.461921] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] return func(*args, **kwargs) [ 1005.461921] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1005.461921] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] raise e [ 1005.461921] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1005.461921] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] nwinfo = self.network_api.allocate_for_instance( [ 1005.461921] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1005.461921] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] created_port_ids = self._update_ports_for_instance( [ 1005.461921] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1005.461921] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] with excutils.save_and_reraise_exception(): [ 1005.461921] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1005.461921] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] self.force_reraise() [ 1005.461921] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1005.462380] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] raise self.value [ 1005.462380] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1005.462380] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] updated_port = self._update_port( [ 1005.462380] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1005.462380] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] _ensure_no_port_binding_failure(port) [ 1005.462380] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1005.462380] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] raise exception.PortBindingFailed(port_id=port['id']) [ 1005.462380] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] nova.exception.PortBindingFailed: Binding failed for port 334799b5-89ef-45c5-b835-f2b50f55d337, please check neutron logs for more information. [ 1005.462380] env[62499]: ERROR nova.compute.manager [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] [ 1005.462380] env[62499]: DEBUG nova.compute.utils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Binding failed for port 334799b5-89ef-45c5-b835-f2b50f55d337, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1005.463385] env[62499]: DEBUG nova.compute.manager [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Build of instance 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2 was re-scheduled: Binding failed for port 334799b5-89ef-45c5-b835-f2b50f55d337, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1005.463787] env[62499]: DEBUG nova.compute.manager [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1005.464016] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "refresh_cache-3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1005.464167] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquired lock "refresh_cache-3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1005.464324] env[62499]: DEBUG nova.network.neutron [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1005.988691] env[62499]: DEBUG nova.network.neutron [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1006.119566] env[62499]: DEBUG nova.network.neutron [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1006.415026] env[62499]: INFO nova.scheduler.client.report [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Deleted allocations for instance 9d9196c8-3e26-4223-ab13-af4bcd31787b [ 1006.623450] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Releasing lock "refresh_cache-3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1006.623689] env[62499]: DEBUG nova.compute.manager [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1006.623878] env[62499]: DEBUG nova.compute.manager [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1006.624059] env[62499]: DEBUG nova.network.neutron [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1006.643988] env[62499]: DEBUG nova.network.neutron [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1006.927956] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b2c92193-b4f4-4fe3-9aa5-d1bb048cb0f5 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "9d9196c8-3e26-4223-ab13-af4bcd31787b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 16.176s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1007.146591] env[62499]: DEBUG nova.network.neutron [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1007.650080] env[62499]: INFO nova.compute.manager [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2] Took 1.03 seconds to deallocate network for instance. [ 1008.259177] env[62499]: DEBUG oslo_concurrency.lockutils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "607a5220-597e-4480-ae4d-150802c6196a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1008.259468] env[62499]: DEBUG oslo_concurrency.lockutils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "607a5220-597e-4480-ae4d-150802c6196a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1008.682850] env[62499]: INFO nova.scheduler.client.report [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Deleted allocations for instance 3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2 [ 1008.762009] env[62499]: DEBUG nova.compute.manager [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1009.190930] env[62499]: DEBUG oslo_concurrency.lockutils [None req-be2c283d-b325-4dca-869f-2c08b53210c3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "3e1fbc16-5a38-4fe4-aa5b-b7982b81e7a2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 16.303s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1009.281657] env[62499]: DEBUG oslo_concurrency.lockutils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1009.281910] env[62499]: DEBUG oslo_concurrency.lockutils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1009.283391] env[62499]: INFO nova.compute.claims [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1010.326729] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-637fe0e5-1794-4c23-84f1-1b9efa9c271a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1010.334360] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-013df000-d986-42fb-9c6e-290dc0f5434f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1010.363490] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bda7a9f9-ae3d-44a4-945b-9197b86c4208 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1010.370011] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3bec7891-05d7-41ef-af02-cbbb095d7b1a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1010.382457] env[62499]: DEBUG nova.compute.provider_tree [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1010.431673] env[62499]: DEBUG oslo_concurrency.lockutils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "b4b02e6d-276d-4271-8d4f-3891cfaec8af" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1010.431892] env[62499]: DEBUG oslo_concurrency.lockutils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "b4b02e6d-276d-4271-8d4f-3891cfaec8af" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1010.885290] env[62499]: DEBUG nova.scheduler.client.report [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1010.933868] env[62499]: DEBUG nova.compute.manager [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1011.390514] env[62499]: DEBUG oslo_concurrency.lockutils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.108s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1011.391062] env[62499]: DEBUG nova.compute.manager [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1011.450808] env[62499]: DEBUG oslo_concurrency.lockutils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1011.451081] env[62499]: DEBUG oslo_concurrency.lockutils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1011.452821] env[62499]: INFO nova.compute.claims [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1011.895355] env[62499]: DEBUG nova.compute.utils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1011.896739] env[62499]: DEBUG nova.compute.manager [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1011.896912] env[62499]: DEBUG nova.network.neutron [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1011.951554] env[62499]: DEBUG nova.policy [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7da3078e8bb5444c96e007cfc2d57952', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c9f95d7e954f4413b01aad519c498967', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 1012.187667] env[62499]: DEBUG nova.network.neutron [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Successfully created port: 4ed1776b-c57d-4da6-a8f3-b15b2561f300 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1012.399584] env[62499]: DEBUG nova.compute.manager [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1012.498236] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0e4f861-32c6-437e-a786-4dbb6c7f2172 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1012.505579] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d6530608-0124-4ccc-af79-f35491e1b57d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1012.535386] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8a0a9352-fc2a-4e47-9ead-4ffd0291d5a8 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1012.542087] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28b197bc-61d6-4866-97f4-1483bdb93f44 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1012.554637] env[62499]: DEBUG nova.compute.provider_tree [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1012.793791] env[62499]: DEBUG nova.compute.manager [req-7807ba20-b8e4-4838-b660-b11dc68d7002 req-9dce6b83-c6d3-4cf5-99ef-68963938cb99 service nova] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Received event network-changed-4ed1776b-c57d-4da6-a8f3-b15b2561f300 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 1012.794078] env[62499]: DEBUG nova.compute.manager [req-7807ba20-b8e4-4838-b660-b11dc68d7002 req-9dce6b83-c6d3-4cf5-99ef-68963938cb99 service nova] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Refreshing instance network info cache due to event network-changed-4ed1776b-c57d-4da6-a8f3-b15b2561f300. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 1012.794633] env[62499]: DEBUG oslo_concurrency.lockutils [req-7807ba20-b8e4-4838-b660-b11dc68d7002 req-9dce6b83-c6d3-4cf5-99ef-68963938cb99 service nova] Acquiring lock "refresh_cache-607a5220-597e-4480-ae4d-150802c6196a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1012.794787] env[62499]: DEBUG oslo_concurrency.lockutils [req-7807ba20-b8e4-4838-b660-b11dc68d7002 req-9dce6b83-c6d3-4cf5-99ef-68963938cb99 service nova] Acquired lock "refresh_cache-607a5220-597e-4480-ae4d-150802c6196a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1012.794952] env[62499]: DEBUG nova.network.neutron [req-7807ba20-b8e4-4838-b660-b11dc68d7002 req-9dce6b83-c6d3-4cf5-99ef-68963938cb99 service nova] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Refreshing network info cache for port 4ed1776b-c57d-4da6-a8f3-b15b2561f300 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1012.966922] env[62499]: ERROR nova.compute.manager [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 4ed1776b-c57d-4da6-a8f3-b15b2561f300, please check neutron logs for more information. [ 1012.966922] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 1012.966922] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1012.966922] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1012.966922] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1012.966922] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1012.966922] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1012.966922] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1012.966922] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1012.966922] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 1012.966922] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1012.966922] env[62499]: ERROR nova.compute.manager raise self.value [ 1012.966922] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1012.966922] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 1012.966922] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1012.966922] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1012.967546] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1012.967546] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1012.967546] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 4ed1776b-c57d-4da6-a8f3-b15b2561f300, please check neutron logs for more information. [ 1012.967546] env[62499]: ERROR nova.compute.manager [ 1012.967546] env[62499]: Traceback (most recent call last): [ 1012.967546] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1012.967546] env[62499]: listener.cb(fileno) [ 1012.967546] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1012.967546] env[62499]: result = function(*args, **kwargs) [ 1012.967546] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1012.967546] env[62499]: return func(*args, **kwargs) [ 1012.967546] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1012.967546] env[62499]: raise e [ 1012.967546] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1012.967546] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 1012.967546] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1012.967546] env[62499]: created_port_ids = self._update_ports_for_instance( [ 1012.967546] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1012.967546] env[62499]: with excutils.save_and_reraise_exception(): [ 1012.967546] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1012.967546] env[62499]: self.force_reraise() [ 1012.967546] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1012.967546] env[62499]: raise self.value [ 1012.967546] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1012.967546] env[62499]: updated_port = self._update_port( [ 1012.967546] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1012.967546] env[62499]: _ensure_no_port_binding_failure(port) [ 1012.967546] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1012.967546] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 1012.968494] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 4ed1776b-c57d-4da6-a8f3-b15b2561f300, please check neutron logs for more information. [ 1012.968494] env[62499]: Removing descriptor: 17 [ 1013.059561] env[62499]: DEBUG nova.scheduler.client.report [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1013.315373] env[62499]: DEBUG nova.network.neutron [req-7807ba20-b8e4-4838-b660-b11dc68d7002 req-9dce6b83-c6d3-4cf5-99ef-68963938cb99 service nova] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1013.388321] env[62499]: DEBUG nova.network.neutron [req-7807ba20-b8e4-4838-b660-b11dc68d7002 req-9dce6b83-c6d3-4cf5-99ef-68963938cb99 service nova] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1013.408139] env[62499]: DEBUG nova.compute.manager [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1013.433170] env[62499]: DEBUG nova.virt.hardware [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1013.433432] env[62499]: DEBUG nova.virt.hardware [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1013.433590] env[62499]: DEBUG nova.virt.hardware [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1013.433770] env[62499]: DEBUG nova.virt.hardware [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1013.433916] env[62499]: DEBUG nova.virt.hardware [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1013.434081] env[62499]: DEBUG nova.virt.hardware [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1013.434296] env[62499]: DEBUG nova.virt.hardware [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1013.434457] env[62499]: DEBUG nova.virt.hardware [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1013.434624] env[62499]: DEBUG nova.virt.hardware [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1013.434785] env[62499]: DEBUG nova.virt.hardware [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1013.435438] env[62499]: DEBUG nova.virt.hardware [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1013.435824] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef3be496-0694-4c86-a9a6-e6c44ab4a003 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1013.443079] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e1b78a7d-4aac-4acd-9524-2e9249e2b312 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1013.456062] env[62499]: ERROR nova.compute.manager [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 4ed1776b-c57d-4da6-a8f3-b15b2561f300, please check neutron logs for more information. [ 1013.456062] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] Traceback (most recent call last): [ 1013.456062] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1013.456062] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] yield resources [ 1013.456062] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1013.456062] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] self.driver.spawn(context, instance, image_meta, [ 1013.456062] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 1013.456062] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1013.456062] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1013.456062] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] vm_ref = self.build_virtual_machine(instance, [ 1013.456062] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1013.456532] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] vif_infos = vmwarevif.get_vif_info(self._session, [ 1013.456532] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1013.456532] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] for vif in network_info: [ 1013.456532] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1013.456532] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] return self._sync_wrapper(fn, *args, **kwargs) [ 1013.456532] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1013.456532] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] self.wait() [ 1013.456532] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1013.456532] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] self[:] = self._gt.wait() [ 1013.456532] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1013.456532] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] return self._exit_event.wait() [ 1013.456532] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1013.456532] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] current.throw(*self._exc) [ 1013.456972] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1013.456972] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] result = function(*args, **kwargs) [ 1013.456972] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1013.456972] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] return func(*args, **kwargs) [ 1013.456972] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1013.456972] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] raise e [ 1013.456972] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1013.456972] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] nwinfo = self.network_api.allocate_for_instance( [ 1013.456972] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1013.456972] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] created_port_ids = self._update_ports_for_instance( [ 1013.456972] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1013.456972] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] with excutils.save_and_reraise_exception(): [ 1013.456972] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1013.457459] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] self.force_reraise() [ 1013.457459] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1013.457459] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] raise self.value [ 1013.457459] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1013.457459] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] updated_port = self._update_port( [ 1013.457459] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1013.457459] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] _ensure_no_port_binding_failure(port) [ 1013.457459] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1013.457459] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] raise exception.PortBindingFailed(port_id=port['id']) [ 1013.457459] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] nova.exception.PortBindingFailed: Binding failed for port 4ed1776b-c57d-4da6-a8f3-b15b2561f300, please check neutron logs for more information. [ 1013.457459] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] [ 1013.457459] env[62499]: INFO nova.compute.manager [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Terminating instance [ 1013.458528] env[62499]: DEBUG oslo_concurrency.lockutils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "refresh_cache-607a5220-597e-4480-ae4d-150802c6196a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1013.564497] env[62499]: DEBUG oslo_concurrency.lockutils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.113s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1013.565058] env[62499]: DEBUG nova.compute.manager [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1013.891163] env[62499]: DEBUG oslo_concurrency.lockutils [req-7807ba20-b8e4-4838-b660-b11dc68d7002 req-9dce6b83-c6d3-4cf5-99ef-68963938cb99 service nova] Releasing lock "refresh_cache-607a5220-597e-4480-ae4d-150802c6196a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1013.891587] env[62499]: DEBUG oslo_concurrency.lockutils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquired lock "refresh_cache-607a5220-597e-4480-ae4d-150802c6196a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1013.891773] env[62499]: DEBUG nova.network.neutron [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1014.072080] env[62499]: DEBUG nova.compute.utils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1014.072887] env[62499]: DEBUG nova.compute.manager [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1014.073065] env[62499]: DEBUG nova.network.neutron [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1014.112823] env[62499]: DEBUG nova.policy [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '02640299eb9c4a65bc80de6242dd2cd5', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f509dfe537d441bcacb1a4aab3607214', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 1014.343513] env[62499]: DEBUG nova.network.neutron [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Successfully created port: ac4cba57-ba76-4654-ad50-36ee9a68f1ef {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1014.408908] env[62499]: DEBUG nova.network.neutron [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1014.480979] env[62499]: DEBUG nova.network.neutron [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1014.577799] env[62499]: DEBUG nova.compute.manager [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1014.817438] env[62499]: DEBUG nova.compute.manager [req-0ff7438b-077d-4bff-a45a-2b62b4ba2e50 req-7c0b7cae-9354-4c4c-8fea-435cf08b46e5 service nova] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Received event network-vif-deleted-4ed1776b-c57d-4da6-a8f3-b15b2561f300 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 1014.983468] env[62499]: DEBUG oslo_concurrency.lockutils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Releasing lock "refresh_cache-607a5220-597e-4480-ae4d-150802c6196a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1014.983903] env[62499]: DEBUG nova.compute.manager [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1014.984124] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1014.984422] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-da49abc3-6866-4672-9d28-962e04fa78e1 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1014.993260] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d072063-6a1e-489e-8fec-e1323015331e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1015.014091] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 607a5220-597e-4480-ae4d-150802c6196a could not be found. [ 1015.014300] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1015.014476] env[62499]: INFO nova.compute.manager [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1015.014709] env[62499]: DEBUG oslo.service.loopingcall [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1015.014921] env[62499]: DEBUG nova.compute.manager [-] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1015.015023] env[62499]: DEBUG nova.network.neutron [-] [instance: 607a5220-597e-4480-ae4d-150802c6196a] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1015.028946] env[62499]: DEBUG nova.network.neutron [-] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1015.268469] env[62499]: ERROR nova.compute.manager [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ac4cba57-ba76-4654-ad50-36ee9a68f1ef, please check neutron logs for more information. [ 1015.268469] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 1015.268469] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1015.268469] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1015.268469] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1015.268469] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1015.268469] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1015.268469] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1015.268469] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1015.268469] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 1015.268469] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1015.268469] env[62499]: ERROR nova.compute.manager raise self.value [ 1015.268469] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1015.268469] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 1015.268469] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1015.268469] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1015.269061] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1015.269061] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1015.269061] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ac4cba57-ba76-4654-ad50-36ee9a68f1ef, please check neutron logs for more information. [ 1015.269061] env[62499]: ERROR nova.compute.manager [ 1015.269061] env[62499]: Traceback (most recent call last): [ 1015.269061] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1015.269061] env[62499]: listener.cb(fileno) [ 1015.269061] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1015.269061] env[62499]: result = function(*args, **kwargs) [ 1015.269061] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1015.269061] env[62499]: return func(*args, **kwargs) [ 1015.269061] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1015.269061] env[62499]: raise e [ 1015.269061] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1015.269061] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 1015.269061] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1015.269061] env[62499]: created_port_ids = self._update_ports_for_instance( [ 1015.269061] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1015.269061] env[62499]: with excutils.save_and_reraise_exception(): [ 1015.269061] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1015.269061] env[62499]: self.force_reraise() [ 1015.269061] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1015.269061] env[62499]: raise self.value [ 1015.269061] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1015.269061] env[62499]: updated_port = self._update_port( [ 1015.269061] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1015.269061] env[62499]: _ensure_no_port_binding_failure(port) [ 1015.269061] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1015.269061] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 1015.269998] env[62499]: nova.exception.PortBindingFailed: Binding failed for port ac4cba57-ba76-4654-ad50-36ee9a68f1ef, please check neutron logs for more information. [ 1015.269998] env[62499]: Removing descriptor: 17 [ 1015.531417] env[62499]: DEBUG nova.network.neutron [-] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1015.586713] env[62499]: DEBUG nova.compute.manager [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1015.610870] env[62499]: DEBUG nova.virt.hardware [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1015.611170] env[62499]: DEBUG nova.virt.hardware [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1015.611340] env[62499]: DEBUG nova.virt.hardware [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1015.611523] env[62499]: DEBUG nova.virt.hardware [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1015.611667] env[62499]: DEBUG nova.virt.hardware [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1015.611812] env[62499]: DEBUG nova.virt.hardware [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1015.612027] env[62499]: DEBUG nova.virt.hardware [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1015.612196] env[62499]: DEBUG nova.virt.hardware [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1015.612382] env[62499]: DEBUG nova.virt.hardware [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1015.612590] env[62499]: DEBUG nova.virt.hardware [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1015.612777] env[62499]: DEBUG nova.virt.hardware [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1015.613602] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-06e5122f-65ab-4475-a2b2-2ba0308f086a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1015.621203] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76c0bd1c-bd15-4a79-bd2c-3f674bc54840 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1015.634323] env[62499]: ERROR nova.compute.manager [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ac4cba57-ba76-4654-ad50-36ee9a68f1ef, please check neutron logs for more information. [ 1015.634323] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Traceback (most recent call last): [ 1015.634323] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1015.634323] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] yield resources [ 1015.634323] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1015.634323] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] self.driver.spawn(context, instance, image_meta, [ 1015.634323] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 1015.634323] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1015.634323] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1015.634323] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] vm_ref = self.build_virtual_machine(instance, [ 1015.634323] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1015.634789] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] vif_infos = vmwarevif.get_vif_info(self._session, [ 1015.634789] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1015.634789] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] for vif in network_info: [ 1015.634789] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1015.634789] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] return self._sync_wrapper(fn, *args, **kwargs) [ 1015.634789] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1015.634789] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] self.wait() [ 1015.634789] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1015.634789] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] self[:] = self._gt.wait() [ 1015.634789] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1015.634789] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] return self._exit_event.wait() [ 1015.634789] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1015.634789] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] current.throw(*self._exc) [ 1015.635246] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1015.635246] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] result = function(*args, **kwargs) [ 1015.635246] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1015.635246] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] return func(*args, **kwargs) [ 1015.635246] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1015.635246] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] raise e [ 1015.635246] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1015.635246] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] nwinfo = self.network_api.allocate_for_instance( [ 1015.635246] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1015.635246] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] created_port_ids = self._update_ports_for_instance( [ 1015.635246] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1015.635246] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] with excutils.save_and_reraise_exception(): [ 1015.635246] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1015.635681] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] self.force_reraise() [ 1015.635681] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1015.635681] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] raise self.value [ 1015.635681] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1015.635681] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] updated_port = self._update_port( [ 1015.635681] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1015.635681] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] _ensure_no_port_binding_failure(port) [ 1015.635681] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1015.635681] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] raise exception.PortBindingFailed(port_id=port['id']) [ 1015.635681] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] nova.exception.PortBindingFailed: Binding failed for port ac4cba57-ba76-4654-ad50-36ee9a68f1ef, please check neutron logs for more information. [ 1015.635681] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] [ 1015.635681] env[62499]: INFO nova.compute.manager [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Terminating instance [ 1015.636787] env[62499]: DEBUG oslo_concurrency.lockutils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "refresh_cache-b4b02e6d-276d-4271-8d4f-3891cfaec8af" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1015.636938] env[62499]: DEBUG oslo_concurrency.lockutils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquired lock "refresh_cache-b4b02e6d-276d-4271-8d4f-3891cfaec8af" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1015.637139] env[62499]: DEBUG nova.network.neutron [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1016.034767] env[62499]: INFO nova.compute.manager [-] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Took 1.02 seconds to deallocate network for instance. [ 1016.037454] env[62499]: DEBUG nova.compute.claims [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1016.037635] env[62499]: DEBUG oslo_concurrency.lockutils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1016.037843] env[62499]: DEBUG oslo_concurrency.lockutils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1016.157280] env[62499]: DEBUG nova.network.neutron [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1016.228798] env[62499]: DEBUG nova.network.neutron [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1016.582684] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67208579-057a-4873-bded-44a5f259e343 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1016.589569] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a424ce2-4557-4fc9-82f6-87c6ffd09205 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1016.618104] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e8c1d4e1-9fab-4c48-9647-ffa20c7a5251 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1016.624704] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6cda1b28-5e38-4a95-8a83-0b14b2192eb2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1016.636866] env[62499]: DEBUG nova.compute.provider_tree [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1016.730876] env[62499]: DEBUG oslo_concurrency.lockutils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Releasing lock "refresh_cache-b4b02e6d-276d-4271-8d4f-3891cfaec8af" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1016.731331] env[62499]: DEBUG nova.compute.manager [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1016.731526] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1016.731801] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-16b039a5-5f19-4a37-ac3f-39f21a608af7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1016.740774] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ac6a517-6d64-4bf3-a7ae-91491128f9f3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1016.760510] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b4b02e6d-276d-4271-8d4f-3891cfaec8af could not be found. [ 1016.760700] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1016.760879] env[62499]: INFO nova.compute.manager [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1016.761129] env[62499]: DEBUG oslo.service.loopingcall [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1016.761337] env[62499]: DEBUG nova.compute.manager [-] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1016.761430] env[62499]: DEBUG nova.network.neutron [-] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1016.775648] env[62499]: DEBUG nova.network.neutron [-] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1016.842671] env[62499]: DEBUG nova.compute.manager [req-31b04587-1134-4cc7-8bde-bdea1dd15510 req-9528b2b0-12a8-4937-bd8c-f18d4c9bc785 service nova] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Received event network-changed-ac4cba57-ba76-4654-ad50-36ee9a68f1ef {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 1016.842807] env[62499]: DEBUG nova.compute.manager [req-31b04587-1134-4cc7-8bde-bdea1dd15510 req-9528b2b0-12a8-4937-bd8c-f18d4c9bc785 service nova] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Refreshing instance network info cache due to event network-changed-ac4cba57-ba76-4654-ad50-36ee9a68f1ef. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 1016.843040] env[62499]: DEBUG oslo_concurrency.lockutils [req-31b04587-1134-4cc7-8bde-bdea1dd15510 req-9528b2b0-12a8-4937-bd8c-f18d4c9bc785 service nova] Acquiring lock "refresh_cache-b4b02e6d-276d-4271-8d4f-3891cfaec8af" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1016.843172] env[62499]: DEBUG oslo_concurrency.lockutils [req-31b04587-1134-4cc7-8bde-bdea1dd15510 req-9528b2b0-12a8-4937-bd8c-f18d4c9bc785 service nova] Acquired lock "refresh_cache-b4b02e6d-276d-4271-8d4f-3891cfaec8af" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1016.843334] env[62499]: DEBUG nova.network.neutron [req-31b04587-1134-4cc7-8bde-bdea1dd15510 req-9528b2b0-12a8-4937-bd8c-f18d4c9bc785 service nova] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Refreshing network info cache for port ac4cba57-ba76-4654-ad50-36ee9a68f1ef {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1017.140284] env[62499]: DEBUG nova.scheduler.client.report [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1017.278129] env[62499]: DEBUG nova.network.neutron [-] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1017.361494] env[62499]: DEBUG nova.network.neutron [req-31b04587-1134-4cc7-8bde-bdea1dd15510 req-9528b2b0-12a8-4937-bd8c-f18d4c9bc785 service nova] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1017.431868] env[62499]: DEBUG nova.network.neutron [req-31b04587-1134-4cc7-8bde-bdea1dd15510 req-9528b2b0-12a8-4937-bd8c-f18d4c9bc785 service nova] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1017.644946] env[62499]: DEBUG oslo_concurrency.lockutils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.607s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1017.645665] env[62499]: ERROR nova.compute.manager [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 4ed1776b-c57d-4da6-a8f3-b15b2561f300, please check neutron logs for more information. [ 1017.645665] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] Traceback (most recent call last): [ 1017.645665] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1017.645665] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] self.driver.spawn(context, instance, image_meta, [ 1017.645665] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 1017.645665] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1017.645665] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1017.645665] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] vm_ref = self.build_virtual_machine(instance, [ 1017.645665] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1017.645665] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] vif_infos = vmwarevif.get_vif_info(self._session, [ 1017.645665] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1017.646083] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] for vif in network_info: [ 1017.646083] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1017.646083] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] return self._sync_wrapper(fn, *args, **kwargs) [ 1017.646083] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1017.646083] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] self.wait() [ 1017.646083] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1017.646083] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] self[:] = self._gt.wait() [ 1017.646083] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1017.646083] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] return self._exit_event.wait() [ 1017.646083] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1017.646083] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] current.throw(*self._exc) [ 1017.646083] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1017.646083] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] result = function(*args, **kwargs) [ 1017.646642] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1017.646642] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] return func(*args, **kwargs) [ 1017.646642] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1017.646642] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] raise e [ 1017.646642] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1017.646642] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] nwinfo = self.network_api.allocate_for_instance( [ 1017.646642] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1017.646642] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] created_port_ids = self._update_ports_for_instance( [ 1017.646642] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1017.646642] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] with excutils.save_and_reraise_exception(): [ 1017.646642] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1017.646642] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] self.force_reraise() [ 1017.646642] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1017.647211] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] raise self.value [ 1017.647211] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1017.647211] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] updated_port = self._update_port( [ 1017.647211] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1017.647211] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] _ensure_no_port_binding_failure(port) [ 1017.647211] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1017.647211] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] raise exception.PortBindingFailed(port_id=port['id']) [ 1017.647211] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] nova.exception.PortBindingFailed: Binding failed for port 4ed1776b-c57d-4da6-a8f3-b15b2561f300, please check neutron logs for more information. [ 1017.647211] env[62499]: ERROR nova.compute.manager [instance: 607a5220-597e-4480-ae4d-150802c6196a] [ 1017.647211] env[62499]: DEBUG nova.compute.utils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Binding failed for port 4ed1776b-c57d-4da6-a8f3-b15b2561f300, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1017.648305] env[62499]: DEBUG nova.compute.manager [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Build of instance 607a5220-597e-4480-ae4d-150802c6196a was re-scheduled: Binding failed for port 4ed1776b-c57d-4da6-a8f3-b15b2561f300, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1017.648755] env[62499]: DEBUG nova.compute.manager [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1017.648949] env[62499]: DEBUG oslo_concurrency.lockutils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "refresh_cache-607a5220-597e-4480-ae4d-150802c6196a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1017.649105] env[62499]: DEBUG oslo_concurrency.lockutils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquired lock "refresh_cache-607a5220-597e-4480-ae4d-150802c6196a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1017.649268] env[62499]: DEBUG nova.network.neutron [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1017.780656] env[62499]: INFO nova.compute.manager [-] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Took 1.02 seconds to deallocate network for instance. [ 1017.782967] env[62499]: DEBUG nova.compute.claims [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1017.783173] env[62499]: DEBUG oslo_concurrency.lockutils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1017.783387] env[62499]: DEBUG oslo_concurrency.lockutils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1017.934636] env[62499]: DEBUG oslo_concurrency.lockutils [req-31b04587-1134-4cc7-8bde-bdea1dd15510 req-9528b2b0-12a8-4937-bd8c-f18d4c9bc785 service nova] Releasing lock "refresh_cache-b4b02e6d-276d-4271-8d4f-3891cfaec8af" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1017.934961] env[62499]: DEBUG nova.compute.manager [req-31b04587-1134-4cc7-8bde-bdea1dd15510 req-9528b2b0-12a8-4937-bd8c-f18d4c9bc785 service nova] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Received event network-vif-deleted-ac4cba57-ba76-4654-ad50-36ee9a68f1ef {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 1018.166544] env[62499]: DEBUG nova.network.neutron [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1018.232684] env[62499]: DEBUG nova.network.neutron [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1018.321907] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8934032a-2587-4ac6-8167-a2de55106fd9 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1018.329125] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67ad2265-fc47-46ba-a155-0bef929cfe36 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1018.358059] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-479cfcd2-9c8f-4149-96e5-7c02c0cf2cf7 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1018.364988] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e8b2ccc-b3e7-460a-a98e-204b3515850b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1018.378903] env[62499]: DEBUG nova.compute.provider_tree [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1018.735556] env[62499]: DEBUG oslo_concurrency.lockutils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Releasing lock "refresh_cache-607a5220-597e-4480-ae4d-150802c6196a" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1018.735977] env[62499]: DEBUG nova.compute.manager [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1018.735977] env[62499]: DEBUG nova.compute.manager [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1018.736149] env[62499]: DEBUG nova.network.neutron [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1018.752101] env[62499]: DEBUG nova.network.neutron [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1018.882011] env[62499]: DEBUG nova.scheduler.client.report [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1019.254346] env[62499]: DEBUG nova.network.neutron [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1019.386977] env[62499]: DEBUG oslo_concurrency.lockutils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.603s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1019.387564] env[62499]: ERROR nova.compute.manager [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ac4cba57-ba76-4654-ad50-36ee9a68f1ef, please check neutron logs for more information. [ 1019.387564] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Traceback (most recent call last): [ 1019.387564] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1019.387564] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] self.driver.spawn(context, instance, image_meta, [ 1019.387564] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 1019.387564] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1019.387564] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1019.387564] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] vm_ref = self.build_virtual_machine(instance, [ 1019.387564] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1019.387564] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] vif_infos = vmwarevif.get_vif_info(self._session, [ 1019.387564] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1019.387945] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] for vif in network_info: [ 1019.387945] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1019.387945] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] return self._sync_wrapper(fn, *args, **kwargs) [ 1019.387945] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1019.387945] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] self.wait() [ 1019.387945] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1019.387945] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] self[:] = self._gt.wait() [ 1019.387945] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1019.387945] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] return self._exit_event.wait() [ 1019.387945] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1019.387945] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] current.throw(*self._exc) [ 1019.387945] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1019.387945] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] result = function(*args, **kwargs) [ 1019.388384] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1019.388384] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] return func(*args, **kwargs) [ 1019.388384] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1019.388384] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] raise e [ 1019.388384] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1019.388384] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] nwinfo = self.network_api.allocate_for_instance( [ 1019.388384] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1019.388384] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] created_port_ids = self._update_ports_for_instance( [ 1019.388384] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1019.388384] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] with excutils.save_and_reraise_exception(): [ 1019.388384] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1019.388384] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] self.force_reraise() [ 1019.388384] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1019.389133] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] raise self.value [ 1019.389133] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1019.389133] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] updated_port = self._update_port( [ 1019.389133] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1019.389133] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] _ensure_no_port_binding_failure(port) [ 1019.389133] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1019.389133] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] raise exception.PortBindingFailed(port_id=port['id']) [ 1019.389133] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] nova.exception.PortBindingFailed: Binding failed for port ac4cba57-ba76-4654-ad50-36ee9a68f1ef, please check neutron logs for more information. [ 1019.389133] env[62499]: ERROR nova.compute.manager [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] [ 1019.389133] env[62499]: DEBUG nova.compute.utils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Binding failed for port ac4cba57-ba76-4654-ad50-36ee9a68f1ef, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1019.390013] env[62499]: DEBUG nova.compute.manager [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Build of instance b4b02e6d-276d-4271-8d4f-3891cfaec8af was re-scheduled: Binding failed for port ac4cba57-ba76-4654-ad50-36ee9a68f1ef, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1019.390474] env[62499]: DEBUG nova.compute.manager [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1019.390657] env[62499]: DEBUG oslo_concurrency.lockutils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "refresh_cache-b4b02e6d-276d-4271-8d4f-3891cfaec8af" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1019.390799] env[62499]: DEBUG oslo_concurrency.lockutils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquired lock "refresh_cache-b4b02e6d-276d-4271-8d4f-3891cfaec8af" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1019.390950] env[62499]: DEBUG nova.network.neutron [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1019.757118] env[62499]: INFO nova.compute.manager [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 607a5220-597e-4480-ae4d-150802c6196a] Took 1.02 seconds to deallocate network for instance. [ 1019.908026] env[62499]: DEBUG nova.network.neutron [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1019.975076] env[62499]: DEBUG nova.network.neutron [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1020.477639] env[62499]: DEBUG oslo_concurrency.lockutils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Releasing lock "refresh_cache-b4b02e6d-276d-4271-8d4f-3891cfaec8af" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1020.477969] env[62499]: DEBUG nova.compute.manager [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1020.478203] env[62499]: DEBUG nova.compute.manager [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1020.478380] env[62499]: DEBUG nova.network.neutron [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1020.492425] env[62499]: DEBUG nova.network.neutron [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1020.786129] env[62499]: INFO nova.scheduler.client.report [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Deleted allocations for instance 607a5220-597e-4480-ae4d-150802c6196a [ 1020.995016] env[62499]: DEBUG nova.network.neutron [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1021.293970] env[62499]: DEBUG oslo_concurrency.lockutils [None req-46bb1efe-5c23-4d52-bd29-24c167cdb74d tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "607a5220-597e-4480-ae4d-150802c6196a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 13.034s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1021.498330] env[62499]: INFO nova.compute.manager [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: b4b02e6d-276d-4271-8d4f-3891cfaec8af] Took 1.02 seconds to deallocate network for instance. [ 1022.520681] env[62499]: DEBUG oslo_concurrency.lockutils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "86fcebad-2773-4e53-b745-2fd6acaf4756" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1022.520681] env[62499]: DEBUG oslo_concurrency.lockutils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "86fcebad-2773-4e53-b745-2fd6acaf4756" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1022.523705] env[62499]: INFO nova.scheduler.client.report [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Deleted allocations for instance b4b02e6d-276d-4271-8d4f-3891cfaec8af [ 1023.023341] env[62499]: DEBUG nova.compute.manager [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1023.030778] env[62499]: DEBUG oslo_concurrency.lockutils [None req-41371a42-7802-4cdc-af6b-98cb6e9ef16b tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "b4b02e6d-276d-4271-8d4f-3891cfaec8af" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 12.599s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1023.541887] env[62499]: DEBUG oslo_concurrency.lockutils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1023.542212] env[62499]: DEBUG oslo_concurrency.lockutils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1023.543979] env[62499]: INFO nova.compute.claims [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1024.596646] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da484628-c76e-4e0e-8ffe-b6b698f4f44a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1024.604297] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4f8ab93-fd6a-4467-96b6-7f91d839b651 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1024.634622] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b4313dd-2016-49b3-a9ae-80b6fa7401c0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1024.641405] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3412a7d-b3a6-4f44-8d2c-872405cd686f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1024.654019] env[62499]: DEBUG nova.compute.provider_tree [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1024.730536] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "e2733e84-0140-4b7f-8a7b-bdbb079d09b5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1024.730761] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "e2733e84-0140-4b7f-8a7b-bdbb079d09b5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1025.156953] env[62499]: DEBUG nova.scheduler.client.report [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1025.233344] env[62499]: DEBUG nova.compute.manager [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1025.662369] env[62499]: DEBUG oslo_concurrency.lockutils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.120s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1025.662934] env[62499]: DEBUG nova.compute.manager [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1025.754029] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1025.754029] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1025.755326] env[62499]: INFO nova.compute.claims [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1026.168224] env[62499]: DEBUG nova.compute.utils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1026.169694] env[62499]: DEBUG nova.compute.manager [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1026.169871] env[62499]: DEBUG nova.network.neutron [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1026.217747] env[62499]: DEBUG nova.policy [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7da3078e8bb5444c96e007cfc2d57952', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c9f95d7e954f4413b01aad519c498967', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 1026.465523] env[62499]: DEBUG nova.network.neutron [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Successfully created port: 69d32c23-6404-40bf-9165-725175531e05 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1026.673468] env[62499]: DEBUG nova.compute.manager [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1026.800271] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45d6ebf0-ff65-445a-9fa2-17b8062cc1c0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1026.807458] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39459650-dac1-4fa9-b802-635e772d4d87 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1026.837571] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60f57f21-6918-4054-8e7e-dbfe29f26cba {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1026.843966] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f10aa811-655c-4b9b-96dd-8d91999c387b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1026.856785] env[62499]: DEBUG nova.compute.provider_tree [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1027.062299] env[62499]: DEBUG nova.compute.manager [req-e60cb2b9-a5fd-4f4f-84ef-dfde856ed665 req-bfaaf89a-3a7e-47e4-bba6-821a6b5d6bc5 service nova] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Received event network-changed-69d32c23-6404-40bf-9165-725175531e05 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 1027.062498] env[62499]: DEBUG nova.compute.manager [req-e60cb2b9-a5fd-4f4f-84ef-dfde856ed665 req-bfaaf89a-3a7e-47e4-bba6-821a6b5d6bc5 service nova] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Refreshing instance network info cache due to event network-changed-69d32c23-6404-40bf-9165-725175531e05. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 1027.062718] env[62499]: DEBUG oslo_concurrency.lockutils [req-e60cb2b9-a5fd-4f4f-84ef-dfde856ed665 req-bfaaf89a-3a7e-47e4-bba6-821a6b5d6bc5 service nova] Acquiring lock "refresh_cache-86fcebad-2773-4e53-b745-2fd6acaf4756" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1027.062900] env[62499]: DEBUG oslo_concurrency.lockutils [req-e60cb2b9-a5fd-4f4f-84ef-dfde856ed665 req-bfaaf89a-3a7e-47e4-bba6-821a6b5d6bc5 service nova] Acquired lock "refresh_cache-86fcebad-2773-4e53-b745-2fd6acaf4756" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1027.062996] env[62499]: DEBUG nova.network.neutron [req-e60cb2b9-a5fd-4f4f-84ef-dfde856ed665 req-bfaaf89a-3a7e-47e4-bba6-821a6b5d6bc5 service nova] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Refreshing network info cache for port 69d32c23-6404-40bf-9165-725175531e05 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1027.227030] env[62499]: ERROR nova.compute.manager [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 69d32c23-6404-40bf-9165-725175531e05, please check neutron logs for more information. [ 1027.227030] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 1027.227030] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1027.227030] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1027.227030] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1027.227030] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1027.227030] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1027.227030] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1027.227030] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1027.227030] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 1027.227030] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1027.227030] env[62499]: ERROR nova.compute.manager raise self.value [ 1027.227030] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1027.227030] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 1027.227030] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1027.227030] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1027.227685] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1027.227685] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1027.227685] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 69d32c23-6404-40bf-9165-725175531e05, please check neutron logs for more information. [ 1027.227685] env[62499]: ERROR nova.compute.manager [ 1027.227685] env[62499]: Traceback (most recent call last): [ 1027.227685] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1027.227685] env[62499]: listener.cb(fileno) [ 1027.227685] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1027.227685] env[62499]: result = function(*args, **kwargs) [ 1027.227685] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1027.227685] env[62499]: return func(*args, **kwargs) [ 1027.227685] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1027.227685] env[62499]: raise e [ 1027.227685] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1027.227685] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 1027.227685] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1027.227685] env[62499]: created_port_ids = self._update_ports_for_instance( [ 1027.227685] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1027.227685] env[62499]: with excutils.save_and_reraise_exception(): [ 1027.227685] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1027.227685] env[62499]: self.force_reraise() [ 1027.227685] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1027.227685] env[62499]: raise self.value [ 1027.227685] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1027.227685] env[62499]: updated_port = self._update_port( [ 1027.227685] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1027.227685] env[62499]: _ensure_no_port_binding_failure(port) [ 1027.227685] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1027.227685] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 1027.228711] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 69d32c23-6404-40bf-9165-725175531e05, please check neutron logs for more information. [ 1027.228711] env[62499]: Removing descriptor: 17 [ 1027.359480] env[62499]: DEBUG nova.scheduler.client.report [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1027.581609] env[62499]: DEBUG nova.network.neutron [req-e60cb2b9-a5fd-4f4f-84ef-dfde856ed665 req-bfaaf89a-3a7e-47e4-bba6-821a6b5d6bc5 service nova] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1027.651044] env[62499]: DEBUG nova.network.neutron [req-e60cb2b9-a5fd-4f4f-84ef-dfde856ed665 req-bfaaf89a-3a7e-47e4-bba6-821a6b5d6bc5 service nova] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1027.684145] env[62499]: DEBUG nova.compute.manager [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1027.707373] env[62499]: DEBUG nova.virt.hardware [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1027.707617] env[62499]: DEBUG nova.virt.hardware [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1027.707772] env[62499]: DEBUG nova.virt.hardware [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1027.707970] env[62499]: DEBUG nova.virt.hardware [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1027.708154] env[62499]: DEBUG nova.virt.hardware [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1027.708313] env[62499]: DEBUG nova.virt.hardware [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1027.708532] env[62499]: DEBUG nova.virt.hardware [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1027.708675] env[62499]: DEBUG nova.virt.hardware [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1027.708838] env[62499]: DEBUG nova.virt.hardware [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1027.708999] env[62499]: DEBUG nova.virt.hardware [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1027.709192] env[62499]: DEBUG nova.virt.hardware [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1027.710080] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28ffb391-d332-40bc-af07-33504bb4c272 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1027.717760] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0605b3a7-ba03-442e-989a-475f372ec68d {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1027.732790] env[62499]: ERROR nova.compute.manager [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 69d32c23-6404-40bf-9165-725175531e05, please check neutron logs for more information. [ 1027.732790] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Traceback (most recent call last): [ 1027.732790] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1027.732790] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] yield resources [ 1027.732790] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1027.732790] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] self.driver.spawn(context, instance, image_meta, [ 1027.732790] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 1027.732790] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1027.732790] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1027.732790] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] vm_ref = self.build_virtual_machine(instance, [ 1027.732790] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1027.733371] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] vif_infos = vmwarevif.get_vif_info(self._session, [ 1027.733371] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1027.733371] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] for vif in network_info: [ 1027.733371] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1027.733371] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] return self._sync_wrapper(fn, *args, **kwargs) [ 1027.733371] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1027.733371] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] self.wait() [ 1027.733371] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1027.733371] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] self[:] = self._gt.wait() [ 1027.733371] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1027.733371] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] return self._exit_event.wait() [ 1027.733371] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1027.733371] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] current.throw(*self._exc) [ 1027.734016] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1027.734016] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] result = function(*args, **kwargs) [ 1027.734016] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1027.734016] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] return func(*args, **kwargs) [ 1027.734016] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1027.734016] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] raise e [ 1027.734016] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1027.734016] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] nwinfo = self.network_api.allocate_for_instance( [ 1027.734016] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1027.734016] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] created_port_ids = self._update_ports_for_instance( [ 1027.734016] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1027.734016] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] with excutils.save_and_reraise_exception(): [ 1027.734016] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1027.734590] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] self.force_reraise() [ 1027.734590] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1027.734590] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] raise self.value [ 1027.734590] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1027.734590] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] updated_port = self._update_port( [ 1027.734590] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1027.734590] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] _ensure_no_port_binding_failure(port) [ 1027.734590] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1027.734590] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] raise exception.PortBindingFailed(port_id=port['id']) [ 1027.734590] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] nova.exception.PortBindingFailed: Binding failed for port 69d32c23-6404-40bf-9165-725175531e05, please check neutron logs for more information. [ 1027.734590] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] [ 1027.734590] env[62499]: INFO nova.compute.manager [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Terminating instance [ 1027.735047] env[62499]: DEBUG oslo_concurrency.lockutils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "refresh_cache-86fcebad-2773-4e53-b745-2fd6acaf4756" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1027.864850] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.111s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1027.865415] env[62499]: DEBUG nova.compute.manager [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1028.153578] env[62499]: DEBUG oslo_concurrency.lockutils [req-e60cb2b9-a5fd-4f4f-84ef-dfde856ed665 req-bfaaf89a-3a7e-47e4-bba6-821a6b5d6bc5 service nova] Releasing lock "refresh_cache-86fcebad-2773-4e53-b745-2fd6acaf4756" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1028.153978] env[62499]: DEBUG oslo_concurrency.lockutils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquired lock "refresh_cache-86fcebad-2773-4e53-b745-2fd6acaf4756" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1028.154178] env[62499]: DEBUG nova.network.neutron [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1028.369898] env[62499]: DEBUG nova.compute.utils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1028.371425] env[62499]: DEBUG nova.compute.manager [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1028.371599] env[62499]: DEBUG nova.network.neutron [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1028.414112] env[62499]: DEBUG nova.policy [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '02640299eb9c4a65bc80de6242dd2cd5', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f509dfe537d441bcacb1a4aab3607214', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 1028.639311] env[62499]: DEBUG nova.network.neutron [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Successfully created port: 28608160-16ec-4af3-93f3-843fe7d0630b {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1028.670927] env[62499]: DEBUG nova.network.neutron [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1028.738907] env[62499]: DEBUG nova.network.neutron [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1028.874938] env[62499]: DEBUG nova.compute.manager [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1029.086157] env[62499]: DEBUG nova.compute.manager [req-b1afcd4e-b518-402a-9a11-27ca2e4c038f req-ff22c388-70f9-42c9-bc64-262676345c88 service nova] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Received event network-vif-deleted-69d32c23-6404-40bf-9165-725175531e05 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 1029.243722] env[62499]: DEBUG oslo_concurrency.lockutils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Releasing lock "refresh_cache-86fcebad-2773-4e53-b745-2fd6acaf4756" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1029.244206] env[62499]: DEBUG nova.compute.manager [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1029.244420] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1029.244697] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-68e0242b-6333-49ba-a7f3-66a7041a0500 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.253933] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-426696d3-e19f-439e-8c8f-363edf30c218 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.275424] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 86fcebad-2773-4e53-b745-2fd6acaf4756 could not be found. [ 1029.275641] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1029.275819] env[62499]: INFO nova.compute.manager [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1029.276074] env[62499]: DEBUG oslo.service.loopingcall [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1029.276293] env[62499]: DEBUG nova.compute.manager [-] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1029.276385] env[62499]: DEBUG nova.network.neutron [-] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1029.290985] env[62499]: DEBUG nova.network.neutron [-] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1029.429361] env[62499]: ERROR nova.compute.manager [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 28608160-16ec-4af3-93f3-843fe7d0630b, please check neutron logs for more information. [ 1029.429361] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 1029.429361] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1029.429361] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1029.429361] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1029.429361] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1029.429361] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1029.429361] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1029.429361] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1029.429361] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 1029.429361] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1029.429361] env[62499]: ERROR nova.compute.manager raise self.value [ 1029.429361] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1029.429361] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 1029.429361] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1029.429361] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1029.430273] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1029.430273] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1029.430273] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 28608160-16ec-4af3-93f3-843fe7d0630b, please check neutron logs for more information. [ 1029.430273] env[62499]: ERROR nova.compute.manager [ 1029.430273] env[62499]: Traceback (most recent call last): [ 1029.430273] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1029.430273] env[62499]: listener.cb(fileno) [ 1029.430273] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1029.430273] env[62499]: result = function(*args, **kwargs) [ 1029.430273] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1029.430273] env[62499]: return func(*args, **kwargs) [ 1029.430273] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1029.430273] env[62499]: raise e [ 1029.430273] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1029.430273] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 1029.430273] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1029.430273] env[62499]: created_port_ids = self._update_ports_for_instance( [ 1029.430273] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1029.430273] env[62499]: with excutils.save_and_reraise_exception(): [ 1029.430273] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1029.430273] env[62499]: self.force_reraise() [ 1029.430273] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1029.430273] env[62499]: raise self.value [ 1029.430273] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1029.430273] env[62499]: updated_port = self._update_port( [ 1029.430273] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1029.430273] env[62499]: _ensure_no_port_binding_failure(port) [ 1029.430273] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1029.430273] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 1029.431167] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 28608160-16ec-4af3-93f3-843fe7d0630b, please check neutron logs for more information. [ 1029.431167] env[62499]: Removing descriptor: 17 [ 1029.793216] env[62499]: DEBUG nova.network.neutron [-] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1029.885146] env[62499]: DEBUG nova.compute.manager [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1029.911973] env[62499]: DEBUG nova.virt.hardware [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1029.912261] env[62499]: DEBUG nova.virt.hardware [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1029.912470] env[62499]: DEBUG nova.virt.hardware [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1029.912674] env[62499]: DEBUG nova.virt.hardware [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1029.912852] env[62499]: DEBUG nova.virt.hardware [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1029.913050] env[62499]: DEBUG nova.virt.hardware [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1029.913303] env[62499]: DEBUG nova.virt.hardware [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1029.913489] env[62499]: DEBUG nova.virt.hardware [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1029.913687] env[62499]: DEBUG nova.virt.hardware [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1029.913888] env[62499]: DEBUG nova.virt.hardware [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1029.914110] env[62499]: DEBUG nova.virt.hardware [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1029.915022] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8f9fc91-38a1-41cd-b586-0dca67d21d00 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.923025] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2821ee29-8f4e-49fb-bfe7-e1ce7a6da83c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.937682] env[62499]: ERROR nova.compute.manager [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 28608160-16ec-4af3-93f3-843fe7d0630b, please check neutron logs for more information. [ 1029.937682] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Traceback (most recent call last): [ 1029.937682] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1029.937682] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] yield resources [ 1029.937682] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1029.937682] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] self.driver.spawn(context, instance, image_meta, [ 1029.937682] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 1029.937682] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1029.937682] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1029.937682] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] vm_ref = self.build_virtual_machine(instance, [ 1029.937682] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1029.938169] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] vif_infos = vmwarevif.get_vif_info(self._session, [ 1029.938169] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1029.938169] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] for vif in network_info: [ 1029.938169] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1029.938169] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] return self._sync_wrapper(fn, *args, **kwargs) [ 1029.938169] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1029.938169] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] self.wait() [ 1029.938169] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1029.938169] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] self[:] = self._gt.wait() [ 1029.938169] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1029.938169] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] return self._exit_event.wait() [ 1029.938169] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1029.938169] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] current.throw(*self._exc) [ 1029.938634] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1029.938634] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] result = function(*args, **kwargs) [ 1029.938634] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1029.938634] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] return func(*args, **kwargs) [ 1029.938634] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1029.938634] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] raise e [ 1029.938634] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1029.938634] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] nwinfo = self.network_api.allocate_for_instance( [ 1029.938634] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1029.938634] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] created_port_ids = self._update_ports_for_instance( [ 1029.938634] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1029.938634] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] with excutils.save_and_reraise_exception(): [ 1029.938634] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1029.939133] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] self.force_reraise() [ 1029.939133] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1029.939133] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] raise self.value [ 1029.939133] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1029.939133] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] updated_port = self._update_port( [ 1029.939133] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1029.939133] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] _ensure_no_port_binding_failure(port) [ 1029.939133] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1029.939133] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] raise exception.PortBindingFailed(port_id=port['id']) [ 1029.939133] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] nova.exception.PortBindingFailed: Binding failed for port 28608160-16ec-4af3-93f3-843fe7d0630b, please check neutron logs for more information. [ 1029.939133] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] [ 1029.939133] env[62499]: INFO nova.compute.manager [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Terminating instance [ 1029.939896] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "refresh_cache-e2733e84-0140-4b7f-8a7b-bdbb079d09b5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1029.940068] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquired lock "refresh_cache-e2733e84-0140-4b7f-8a7b-bdbb079d09b5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1029.940262] env[62499]: DEBUG nova.network.neutron [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1030.296354] env[62499]: INFO nova.compute.manager [-] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Took 1.02 seconds to deallocate network for instance. [ 1030.298689] env[62499]: DEBUG nova.compute.claims [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1030.298874] env[62499]: DEBUG oslo_concurrency.lockutils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1030.299114] env[62499]: DEBUG oslo_concurrency.lockutils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1030.459372] env[62499]: DEBUG nova.network.neutron [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1030.545145] env[62499]: DEBUG nova.network.neutron [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1030.854858] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d97153e-210f-437b-acb1-0003dc3ba4c5 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1030.862299] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9eed8082-3b7e-497c-a436-ca99c1b722ac {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1030.891234] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-688b91af-b8a6-4219-a238-c17a1d06b1d3 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1030.897904] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e1bc503-ef90-4db1-9c66-302a77cf9d16 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1030.910531] env[62499]: DEBUG nova.compute.provider_tree [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1031.048313] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Releasing lock "refresh_cache-e2733e84-0140-4b7f-8a7b-bdbb079d09b5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1031.048727] env[62499]: DEBUG nova.compute.manager [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1031.048915] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1031.049541] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-0722e51a-0a45-4aa8-aa80-f3e7066645ef {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1031.060055] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7dbbc1a9-1f4c-4744-9cd0-c36d3b2ce169 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1031.081774] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance e2733e84-0140-4b7f-8a7b-bdbb079d09b5 could not be found. [ 1031.081969] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1031.082177] env[62499]: INFO nova.compute.manager [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1031.082413] env[62499]: DEBUG oslo.service.loopingcall [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1031.082616] env[62499]: DEBUG nova.compute.manager [-] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1031.082711] env[62499]: DEBUG nova.network.neutron [-] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1031.101551] env[62499]: DEBUG nova.network.neutron [-] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1031.113695] env[62499]: DEBUG nova.compute.manager [req-b5fb590d-756b-462b-b2aa-96f0569ab91e req-706b4af8-d0d5-4429-b2b1-059682e55b81 service nova] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Received event network-changed-28608160-16ec-4af3-93f3-843fe7d0630b {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 1031.113893] env[62499]: DEBUG nova.compute.manager [req-b5fb590d-756b-462b-b2aa-96f0569ab91e req-706b4af8-d0d5-4429-b2b1-059682e55b81 service nova] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Refreshing instance network info cache due to event network-changed-28608160-16ec-4af3-93f3-843fe7d0630b. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 1031.114119] env[62499]: DEBUG oslo_concurrency.lockutils [req-b5fb590d-756b-462b-b2aa-96f0569ab91e req-706b4af8-d0d5-4429-b2b1-059682e55b81 service nova] Acquiring lock "refresh_cache-e2733e84-0140-4b7f-8a7b-bdbb079d09b5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1031.114212] env[62499]: DEBUG oslo_concurrency.lockutils [req-b5fb590d-756b-462b-b2aa-96f0569ab91e req-706b4af8-d0d5-4429-b2b1-059682e55b81 service nova] Acquired lock "refresh_cache-e2733e84-0140-4b7f-8a7b-bdbb079d09b5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1031.114369] env[62499]: DEBUG nova.network.neutron [req-b5fb590d-756b-462b-b2aa-96f0569ab91e req-706b4af8-d0d5-4429-b2b1-059682e55b81 service nova] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Refreshing network info cache for port 28608160-16ec-4af3-93f3-843fe7d0630b {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1031.413344] env[62499]: DEBUG nova.scheduler.client.report [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1031.603594] env[62499]: DEBUG nova.network.neutron [-] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1031.630600] env[62499]: DEBUG nova.network.neutron [req-b5fb590d-756b-462b-b2aa-96f0569ab91e req-706b4af8-d0d5-4429-b2b1-059682e55b81 service nova] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1031.699448] env[62499]: DEBUG nova.network.neutron [req-b5fb590d-756b-462b-b2aa-96f0569ab91e req-706b4af8-d0d5-4429-b2b1-059682e55b81 service nova] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1031.918648] env[62499]: DEBUG oslo_concurrency.lockutils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.619s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1031.919252] env[62499]: ERROR nova.compute.manager [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 69d32c23-6404-40bf-9165-725175531e05, please check neutron logs for more information. [ 1031.919252] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Traceback (most recent call last): [ 1031.919252] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1031.919252] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] self.driver.spawn(context, instance, image_meta, [ 1031.919252] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 1031.919252] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1031.919252] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1031.919252] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] vm_ref = self.build_virtual_machine(instance, [ 1031.919252] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1031.919252] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] vif_infos = vmwarevif.get_vif_info(self._session, [ 1031.919252] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1031.919663] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] for vif in network_info: [ 1031.919663] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1031.919663] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] return self._sync_wrapper(fn, *args, **kwargs) [ 1031.919663] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1031.919663] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] self.wait() [ 1031.919663] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1031.919663] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] self[:] = self._gt.wait() [ 1031.919663] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1031.919663] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] return self._exit_event.wait() [ 1031.919663] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1031.919663] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] current.throw(*self._exc) [ 1031.919663] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1031.919663] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] result = function(*args, **kwargs) [ 1031.920137] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1031.920137] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] return func(*args, **kwargs) [ 1031.920137] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1031.920137] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] raise e [ 1031.920137] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1031.920137] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] nwinfo = self.network_api.allocate_for_instance( [ 1031.920137] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1031.920137] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] created_port_ids = self._update_ports_for_instance( [ 1031.920137] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1031.920137] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] with excutils.save_and_reraise_exception(): [ 1031.920137] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1031.920137] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] self.force_reraise() [ 1031.920137] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1031.920740] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] raise self.value [ 1031.920740] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1031.920740] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] updated_port = self._update_port( [ 1031.920740] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1031.920740] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] _ensure_no_port_binding_failure(port) [ 1031.920740] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1031.920740] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] raise exception.PortBindingFailed(port_id=port['id']) [ 1031.920740] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] nova.exception.PortBindingFailed: Binding failed for port 69d32c23-6404-40bf-9165-725175531e05, please check neutron logs for more information. [ 1031.920740] env[62499]: ERROR nova.compute.manager [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] [ 1031.920740] env[62499]: DEBUG nova.compute.utils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Binding failed for port 69d32c23-6404-40bf-9165-725175531e05, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1031.921619] env[62499]: DEBUG nova.compute.manager [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Build of instance 86fcebad-2773-4e53-b745-2fd6acaf4756 was re-scheduled: Binding failed for port 69d32c23-6404-40bf-9165-725175531e05, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1031.922027] env[62499]: DEBUG nova.compute.manager [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1031.922255] env[62499]: DEBUG oslo_concurrency.lockutils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquiring lock "refresh_cache-86fcebad-2773-4e53-b745-2fd6acaf4756" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1031.922405] env[62499]: DEBUG oslo_concurrency.lockutils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Acquired lock "refresh_cache-86fcebad-2773-4e53-b745-2fd6acaf4756" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1031.922560] env[62499]: DEBUG nova.network.neutron [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1032.106200] env[62499]: INFO nova.compute.manager [-] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Took 1.02 seconds to deallocate network for instance. [ 1032.108378] env[62499]: DEBUG nova.compute.claims [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1032.108547] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1032.108754] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1032.201869] env[62499]: DEBUG oslo_concurrency.lockutils [req-b5fb590d-756b-462b-b2aa-96f0569ab91e req-706b4af8-d0d5-4429-b2b1-059682e55b81 service nova] Releasing lock "refresh_cache-e2733e84-0140-4b7f-8a7b-bdbb079d09b5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1032.202117] env[62499]: DEBUG nova.compute.manager [req-b5fb590d-756b-462b-b2aa-96f0569ab91e req-706b4af8-d0d5-4429-b2b1-059682e55b81 service nova] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Received event network-vif-deleted-28608160-16ec-4af3-93f3-843fe7d0630b {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 1032.439579] env[62499]: DEBUG nova.network.neutron [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1032.518335] env[62499]: DEBUG nova.network.neutron [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1032.651053] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b122c3f-cfde-4e87-85bc-668a6d99ab4a {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1032.658835] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-46ee54f1-0acf-4871-b6d1-09ba45eaec71 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1032.689154] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6863745f-558b-4aff-ac1a-f82012b78856 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1032.695734] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5833e28f-539a-42a6-b879-bb75962f2ea2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1032.708096] env[62499]: DEBUG nova.compute.provider_tree [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1033.021278] env[62499]: DEBUG oslo_concurrency.lockutils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Releasing lock "refresh_cache-86fcebad-2773-4e53-b745-2fd6acaf4756" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1033.021640] env[62499]: DEBUG nova.compute.manager [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1033.021708] env[62499]: DEBUG nova.compute.manager [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1033.021848] env[62499]: DEBUG nova.network.neutron [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1033.037900] env[62499]: DEBUG nova.network.neutron [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1033.211387] env[62499]: DEBUG nova.scheduler.client.report [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1033.540495] env[62499]: DEBUG nova.network.neutron [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1033.715896] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.607s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1033.716543] env[62499]: ERROR nova.compute.manager [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 28608160-16ec-4af3-93f3-843fe7d0630b, please check neutron logs for more information. [ 1033.716543] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Traceback (most recent call last): [ 1033.716543] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1033.716543] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] self.driver.spawn(context, instance, image_meta, [ 1033.716543] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 1033.716543] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1033.716543] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1033.716543] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] vm_ref = self.build_virtual_machine(instance, [ 1033.716543] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1033.716543] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] vif_infos = vmwarevif.get_vif_info(self._session, [ 1033.716543] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1033.716977] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] for vif in network_info: [ 1033.716977] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1033.716977] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] return self._sync_wrapper(fn, *args, **kwargs) [ 1033.716977] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1033.716977] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] self.wait() [ 1033.716977] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1033.716977] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] self[:] = self._gt.wait() [ 1033.716977] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1033.716977] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] return self._exit_event.wait() [ 1033.716977] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1033.716977] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] current.throw(*self._exc) [ 1033.716977] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1033.716977] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] result = function(*args, **kwargs) [ 1033.717362] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1033.717362] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] return func(*args, **kwargs) [ 1033.717362] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1033.717362] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] raise e [ 1033.717362] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1033.717362] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] nwinfo = self.network_api.allocate_for_instance( [ 1033.717362] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1033.717362] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] created_port_ids = self._update_ports_for_instance( [ 1033.717362] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1033.717362] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] with excutils.save_and_reraise_exception(): [ 1033.717362] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1033.717362] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] self.force_reraise() [ 1033.717362] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1033.717725] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] raise self.value [ 1033.717725] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1033.717725] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] updated_port = self._update_port( [ 1033.717725] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1033.717725] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] _ensure_no_port_binding_failure(port) [ 1033.717725] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1033.717725] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] raise exception.PortBindingFailed(port_id=port['id']) [ 1033.717725] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] nova.exception.PortBindingFailed: Binding failed for port 28608160-16ec-4af3-93f3-843fe7d0630b, please check neutron logs for more information. [ 1033.717725] env[62499]: ERROR nova.compute.manager [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] [ 1033.717725] env[62499]: DEBUG nova.compute.utils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Binding failed for port 28608160-16ec-4af3-93f3-843fe7d0630b, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1033.718837] env[62499]: DEBUG nova.compute.manager [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Build of instance e2733e84-0140-4b7f-8a7b-bdbb079d09b5 was re-scheduled: Binding failed for port 28608160-16ec-4af3-93f3-843fe7d0630b, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1033.719250] env[62499]: DEBUG nova.compute.manager [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1033.719473] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "refresh_cache-e2733e84-0140-4b7f-8a7b-bdbb079d09b5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1033.719624] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquired lock "refresh_cache-e2733e84-0140-4b7f-8a7b-bdbb079d09b5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1033.719782] env[62499]: DEBUG nova.network.neutron [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1034.043455] env[62499]: INFO nova.compute.manager [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] [instance: 86fcebad-2773-4e53-b745-2fd6acaf4756] Took 1.02 seconds to deallocate network for instance. [ 1034.236711] env[62499]: DEBUG nova.network.neutron [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1034.304366] env[62499]: DEBUG nova.network.neutron [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1034.806468] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Releasing lock "refresh_cache-e2733e84-0140-4b7f-8a7b-bdbb079d09b5" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1034.806710] env[62499]: DEBUG nova.compute.manager [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1034.806892] env[62499]: DEBUG nova.compute.manager [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1034.807076] env[62499]: DEBUG nova.network.neutron [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1034.822057] env[62499]: DEBUG nova.network.neutron [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1035.070344] env[62499]: INFO nova.scheduler.client.report [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Deleted allocations for instance 86fcebad-2773-4e53-b745-2fd6acaf4756 [ 1035.324374] env[62499]: DEBUG nova.network.neutron [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1035.578461] env[62499]: DEBUG oslo_concurrency.lockutils [None req-483b3412-9b53-4902-8762-0f8b053c02b9 tempest-DeleteServersTestJSON-1536511657 tempest-DeleteServersTestJSON-1536511657-project-member] Lock "86fcebad-2773-4e53-b745-2fd6acaf4756" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 13.058s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1035.826888] env[62499]: INFO nova.compute.manager [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: e2733e84-0140-4b7f-8a7b-bdbb079d09b5] Took 1.02 seconds to deallocate network for instance. [ 1036.857933] env[62499]: INFO nova.scheduler.client.report [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Deleted allocations for instance e2733e84-0140-4b7f-8a7b-bdbb079d09b5 [ 1037.369552] env[62499]: DEBUG oslo_concurrency.lockutils [None req-67a0b9ec-bc6d-4e10-a461-fd1106abccd3 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "e2733e84-0140-4b7f-8a7b-bdbb079d09b5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 12.639s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1039.011411] env[62499]: DEBUG oslo_concurrency.lockutils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "6cdc499b-8441-48c6-a6e7-07383e5746db" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1039.011714] env[62499]: DEBUG oslo_concurrency.lockutils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "6cdc499b-8441-48c6-a6e7-07383e5746db" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1039.513864] env[62499]: DEBUG nova.compute.manager [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Starting instance... {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1040.035358] env[62499]: DEBUG oslo_concurrency.lockutils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1040.035625] env[62499]: DEBUG oslo_concurrency.lockutils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1040.037074] env[62499]: INFO nova.compute.claims [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1041.072102] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec4eec74-7f05-4dc0-b11a-d5ce0b67d04b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.079713] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98803a12-703c-4ca0-90a2-c717b7f83b29 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.109837] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a29154c-b7b9-4580-9827-be72ff53e8fd {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.117687] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-449376d7-2268-4e18-9b78-79340e654229 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1041.130657] env[62499]: DEBUG nova.compute.provider_tree [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1041.633980] env[62499]: DEBUG nova.scheduler.client.report [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1042.138484] env[62499]: DEBUG oslo_concurrency.lockutils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.103s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1042.139055] env[62499]: DEBUG nova.compute.manager [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Start building networks asynchronously for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1042.644241] env[62499]: DEBUG nova.compute.utils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Using /dev/sd instead of None {{(pid=62499) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1042.645860] env[62499]: DEBUG nova.compute.manager [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Allocating IP information in the background. {{(pid=62499) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1042.645860] env[62499]: DEBUG nova.network.neutron [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] allocate_for_instance() {{(pid=62499) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1205}} [ 1042.696560] env[62499]: DEBUG nova.policy [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '02640299eb9c4a65bc80de6242dd2cd5', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f509dfe537d441bcacb1a4aab3607214', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62499) authorize /opt/stack/nova/nova/policy.py:201}} [ 1042.941377] env[62499]: DEBUG nova.network.neutron [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Successfully created port: 15adb248-7814-4c18-b4d0-e725f0cb79e3 {{(pid=62499) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1043.148597] env[62499]: DEBUG nova.compute.manager [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Start building block device mappings for instance. {{(pid=62499) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1043.528361] env[62499]: DEBUG nova.compute.manager [req-260afdbe-0e22-4aa2-8a29-38fe4da14a3c req-c679e7a9-27b2-4c54-bad4-81845996fee2 service nova] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Received event network-changed-15adb248-7814-4c18-b4d0-e725f0cb79e3 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 1043.528361] env[62499]: DEBUG nova.compute.manager [req-260afdbe-0e22-4aa2-8a29-38fe4da14a3c req-c679e7a9-27b2-4c54-bad4-81845996fee2 service nova] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Refreshing instance network info cache due to event network-changed-15adb248-7814-4c18-b4d0-e725f0cb79e3. {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11136}} [ 1043.528361] env[62499]: DEBUG oslo_concurrency.lockutils [req-260afdbe-0e22-4aa2-8a29-38fe4da14a3c req-c679e7a9-27b2-4c54-bad4-81845996fee2 service nova] Acquiring lock "refresh_cache-6cdc499b-8441-48c6-a6e7-07383e5746db" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1043.528361] env[62499]: DEBUG oslo_concurrency.lockutils [req-260afdbe-0e22-4aa2-8a29-38fe4da14a3c req-c679e7a9-27b2-4c54-bad4-81845996fee2 service nova] Acquired lock "refresh_cache-6cdc499b-8441-48c6-a6e7-07383e5746db" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1043.528361] env[62499]: DEBUG nova.network.neutron [req-260afdbe-0e22-4aa2-8a29-38fe4da14a3c req-c679e7a9-27b2-4c54-bad4-81845996fee2 service nova] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Refreshing network info cache for port 15adb248-7814-4c18-b4d0-e725f0cb79e3 {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2064}} [ 1043.699686] env[62499]: ERROR nova.compute.manager [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 15adb248-7814-4c18-b4d0-e725f0cb79e3, please check neutron logs for more information. [ 1043.699686] env[62499]: ERROR nova.compute.manager Traceback (most recent call last): [ 1043.699686] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1043.699686] env[62499]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1043.699686] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1043.699686] env[62499]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1043.699686] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1043.699686] env[62499]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1043.699686] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1043.699686] env[62499]: ERROR nova.compute.manager self.force_reraise() [ 1043.699686] env[62499]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1043.699686] env[62499]: ERROR nova.compute.manager raise self.value [ 1043.699686] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1043.699686] env[62499]: ERROR nova.compute.manager updated_port = self._update_port( [ 1043.699686] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1043.699686] env[62499]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1043.700313] env[62499]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1043.700313] env[62499]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1043.700313] env[62499]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 15adb248-7814-4c18-b4d0-e725f0cb79e3, please check neutron logs for more information. [ 1043.700313] env[62499]: ERROR nova.compute.manager [ 1043.700313] env[62499]: Traceback (most recent call last): [ 1043.700313] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1043.700313] env[62499]: listener.cb(fileno) [ 1043.700313] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1043.700313] env[62499]: result = function(*args, **kwargs) [ 1043.700313] env[62499]: File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1043.700313] env[62499]: return func(*args, **kwargs) [ 1043.700313] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1043.700313] env[62499]: raise e [ 1043.700313] env[62499]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1043.700313] env[62499]: nwinfo = self.network_api.allocate_for_instance( [ 1043.700313] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1043.700313] env[62499]: created_port_ids = self._update_ports_for_instance( [ 1043.700313] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1043.700313] env[62499]: with excutils.save_and_reraise_exception(): [ 1043.700313] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1043.700313] env[62499]: self.force_reraise() [ 1043.700313] env[62499]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1043.700313] env[62499]: raise self.value [ 1043.700313] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1043.700313] env[62499]: updated_port = self._update_port( [ 1043.700313] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1043.700313] env[62499]: _ensure_no_port_binding_failure(port) [ 1043.700313] env[62499]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1043.700313] env[62499]: raise exception.PortBindingFailed(port_id=port['id']) [ 1043.701324] env[62499]: nova.exception.PortBindingFailed: Binding failed for port 15adb248-7814-4c18-b4d0-e725f0cb79e3, please check neutron logs for more information. [ 1043.701324] env[62499]: Removing descriptor: 17 [ 1044.045727] env[62499]: DEBUG nova.network.neutron [req-260afdbe-0e22-4aa2-8a29-38fe4da14a3c req-c679e7a9-27b2-4c54-bad4-81845996fee2 service nova] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1044.116438] env[62499]: DEBUG nova.network.neutron [req-260afdbe-0e22-4aa2-8a29-38fe4da14a3c req-c679e7a9-27b2-4c54-bad4-81845996fee2 service nova] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1044.157949] env[62499]: DEBUG nova.compute.manager [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Start spawning the instance on the hypervisor. {{(pid=62499) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1044.182090] env[62499]: DEBUG nova.virt.hardware [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-22T08:31:36Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-22T08:31:19Z,direct_url=,disk_format='vmdk',id=0422123c-16ce-4c9d-b1c6-0281fe0e54ad,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='a2297e59b1a04e9e93e445653234ac91',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-22T08:31:20Z,virtual_size=,visibility=), allow threads: False {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1044.182338] env[62499]: DEBUG nova.virt.hardware [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Flavor limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1044.182530] env[62499]: DEBUG nova.virt.hardware [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Image limits 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1044.182734] env[62499]: DEBUG nova.virt.hardware [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Flavor pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1044.182877] env[62499]: DEBUG nova.virt.hardware [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Image pref 0:0:0 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1044.183030] env[62499]: DEBUG nova.virt.hardware [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62499) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1044.183241] env[62499]: DEBUG nova.virt.hardware [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1044.183397] env[62499]: DEBUG nova.virt.hardware [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1044.183560] env[62499]: DEBUG nova.virt.hardware [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Got 1 possible topologies {{(pid=62499) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1044.183718] env[62499]: DEBUG nova.virt.hardware [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1044.183916] env[62499]: DEBUG nova.virt.hardware [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62499) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1044.184780] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55888efd-8ccc-49b7-b504-e6700c11df9c {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1044.192843] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97836751-1fc5-4c36-8671-56c768c9af8f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1044.206251] env[62499]: ERROR nova.compute.manager [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 15adb248-7814-4c18-b4d0-e725f0cb79e3, please check neutron logs for more information. [ 1044.206251] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Traceback (most recent call last): [ 1044.206251] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1044.206251] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] yield resources [ 1044.206251] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1044.206251] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] self.driver.spawn(context, instance, image_meta, [ 1044.206251] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 1044.206251] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1044.206251] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1044.206251] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] vm_ref = self.build_virtual_machine(instance, [ 1044.206251] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1044.206894] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] vif_infos = vmwarevif.get_vif_info(self._session, [ 1044.206894] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1044.206894] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] for vif in network_info: [ 1044.206894] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1044.206894] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] return self._sync_wrapper(fn, *args, **kwargs) [ 1044.206894] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1044.206894] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] self.wait() [ 1044.206894] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1044.206894] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] self[:] = self._gt.wait() [ 1044.206894] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1044.206894] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] return self._exit_event.wait() [ 1044.206894] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1044.206894] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] current.throw(*self._exc) [ 1044.207553] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1044.207553] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] result = function(*args, **kwargs) [ 1044.207553] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1044.207553] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] return func(*args, **kwargs) [ 1044.207553] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1044.207553] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] raise e [ 1044.207553] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1044.207553] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] nwinfo = self.network_api.allocate_for_instance( [ 1044.207553] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1044.207553] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] created_port_ids = self._update_ports_for_instance( [ 1044.207553] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1044.207553] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] with excutils.save_and_reraise_exception(): [ 1044.207553] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1044.208510] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] self.force_reraise() [ 1044.208510] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1044.208510] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] raise self.value [ 1044.208510] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1044.208510] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] updated_port = self._update_port( [ 1044.208510] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1044.208510] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] _ensure_no_port_binding_failure(port) [ 1044.208510] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1044.208510] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] raise exception.PortBindingFailed(port_id=port['id']) [ 1044.208510] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] nova.exception.PortBindingFailed: Binding failed for port 15adb248-7814-4c18-b4d0-e725f0cb79e3, please check neutron logs for more information. [ 1044.208510] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] [ 1044.208510] env[62499]: INFO nova.compute.manager [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Terminating instance [ 1044.208878] env[62499]: DEBUG oslo_concurrency.lockutils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "refresh_cache-6cdc499b-8441-48c6-a6e7-07383e5746db" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1044.619112] env[62499]: DEBUG oslo_concurrency.lockutils [req-260afdbe-0e22-4aa2-8a29-38fe4da14a3c req-c679e7a9-27b2-4c54-bad4-81845996fee2 service nova] Releasing lock "refresh_cache-6cdc499b-8441-48c6-a6e7-07383e5746db" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1044.619500] env[62499]: DEBUG oslo_concurrency.lockutils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquired lock "refresh_cache-6cdc499b-8441-48c6-a6e7-07383e5746db" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1044.619692] env[62499]: DEBUG nova.network.neutron [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1045.138150] env[62499]: DEBUG nova.network.neutron [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1045.202118] env[62499]: DEBUG nova.network.neutron [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1045.551833] env[62499]: DEBUG nova.compute.manager [req-9eac83a7-7885-478f-9087-298b4d8afcdc req-b99179a1-eb48-4200-acf6-892c74c5dfdb service nova] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Received event network-vif-deleted-15adb248-7814-4c18-b4d0-e725f0cb79e3 {{(pid=62499) external_instance_event /opt/stack/nova/nova/compute/manager.py:11131}} [ 1045.704889] env[62499]: DEBUG oslo_concurrency.lockutils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Releasing lock "refresh_cache-6cdc499b-8441-48c6-a6e7-07383e5746db" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1045.705350] env[62499]: DEBUG nova.compute.manager [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Start destroying the instance on the hypervisor. {{(pid=62499) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1045.705544] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Destroying instance {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1141}} [ 1045.705846] env[62499]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e9870227-1ff9-4e44-aeaa-1f187986a7d0 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1045.715012] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4efe2cfd-e35c-46fc-9e07-ff861eb38b41 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1045.734982] env[62499]: WARNING nova.virt.vmwareapi.vmops [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6cdc499b-8441-48c6-a6e7-07383e5746db could not be found. [ 1045.735192] env[62499]: DEBUG nova.virt.vmwareapi.vmops [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Instance destroyed {{(pid=62499) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1143}} [ 1045.735372] env[62499]: INFO nova.compute.manager [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1045.735611] env[62499]: DEBUG oslo.service.loopingcall [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62499) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1045.735853] env[62499]: DEBUG nova.compute.manager [-] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1045.735949] env[62499]: DEBUG nova.network.neutron [-] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1045.750462] env[62499]: DEBUG nova.network.neutron [-] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1046.253290] env[62499]: DEBUG nova.network.neutron [-] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1046.755649] env[62499]: INFO nova.compute.manager [-] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Took 1.02 seconds to deallocate network for instance. [ 1046.758011] env[62499]: DEBUG nova.compute.claims [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Aborting claim: {{(pid=62499) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1046.758212] env[62499]: DEBUG oslo_concurrency.lockutils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1046.758433] env[62499]: DEBUG oslo_concurrency.lockutils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1047.293775] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90f07394-0839-4a9d-abde-f3f2ae86a5cc {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1047.301735] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-130c3830-7a0e-4eb1-94d8-67444c25e658 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1047.330022] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12031bc8-13d9-48bf-ab51-561994f28b5e {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1047.336957] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d92b76a-9c48-410e-b284-5b2f7b6bba20 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1047.349298] env[62499]: DEBUG nova.compute.provider_tree [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1047.852238] env[62499]: DEBUG nova.scheduler.client.report [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1048.357718] env[62499]: DEBUG oslo_concurrency.lockutils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.599s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1048.358379] env[62499]: ERROR nova.compute.manager [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 15adb248-7814-4c18-b4d0-e725f0cb79e3, please check neutron logs for more information. [ 1048.358379] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Traceback (most recent call last): [ 1048.358379] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1048.358379] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] self.driver.spawn(context, instance, image_meta, [ 1048.358379] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 546, in spawn [ 1048.358379] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1048.358379] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1048.358379] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] vm_ref = self.build_virtual_machine(instance, [ 1048.358379] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1048.358379] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] vif_infos = vmwarevif.get_vif_info(self._session, [ 1048.358379] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1048.358806] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] for vif in network_info: [ 1048.358806] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1048.358806] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] return self._sync_wrapper(fn, *args, **kwargs) [ 1048.358806] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1048.358806] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] self.wait() [ 1048.358806] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1048.358806] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] self[:] = self._gt.wait() [ 1048.358806] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1048.358806] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] return self._exit_event.wait() [ 1048.358806] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1048.358806] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] current.throw(*self._exc) [ 1048.358806] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1048.358806] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] result = function(*args, **kwargs) [ 1048.359242] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/utils.py", line 663, in context_wrapper [ 1048.359242] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] return func(*args, **kwargs) [ 1048.359242] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1048.359242] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] raise e [ 1048.359242] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1048.359242] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] nwinfo = self.network_api.allocate_for_instance( [ 1048.359242] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/network/neutron.py", line 1278, in allocate_for_instance [ 1048.359242] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] created_port_ids = self._update_ports_for_instance( [ 1048.359242] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/network/neutron.py", line 1414, in _update_ports_for_instance [ 1048.359242] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] with excutils.save_and_reraise_exception(): [ 1048.359242] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1048.359242] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] self.force_reraise() [ 1048.359242] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1048.359815] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] raise self.value [ 1048.359815] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/network/neutron.py", line 1389, in _update_ports_for_instance [ 1048.359815] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] updated_port = self._update_port( [ 1048.359815] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1048.359815] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] _ensure_no_port_binding_failure(port) [ 1048.359815] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1048.359815] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] raise exception.PortBindingFailed(port_id=port['id']) [ 1048.359815] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] nova.exception.PortBindingFailed: Binding failed for port 15adb248-7814-4c18-b4d0-e725f0cb79e3, please check neutron logs for more information. [ 1048.359815] env[62499]: ERROR nova.compute.manager [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] [ 1048.359815] env[62499]: DEBUG nova.compute.utils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Binding failed for port 15adb248-7814-4c18-b4d0-e725f0cb79e3, please check neutron logs for more information. {{(pid=62499) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1048.360943] env[62499]: DEBUG nova.compute.manager [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Build of instance 6cdc499b-8441-48c6-a6e7-07383e5746db was re-scheduled: Binding failed for port 15adb248-7814-4c18-b4d0-e725f0cb79e3, please check neutron logs for more information. {{(pid=62499) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1048.361368] env[62499]: DEBUG nova.compute.manager [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Unplugging VIFs for instance {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1048.361594] env[62499]: DEBUG oslo_concurrency.lockutils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquiring lock "refresh_cache-6cdc499b-8441-48c6-a6e7-07383e5746db" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1048.361760] env[62499]: DEBUG oslo_concurrency.lockutils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Acquired lock "refresh_cache-6cdc499b-8441-48c6-a6e7-07383e5746db" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1048.361914] env[62499]: DEBUG nova.network.neutron [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Building network info cache for instance {{(pid=62499) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2067}} [ 1048.878759] env[62499]: DEBUG nova.network.neutron [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1048.948010] env[62499]: DEBUG nova.network.neutron [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1049.450316] env[62499]: DEBUG oslo_concurrency.lockutils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Releasing lock "refresh_cache-6cdc499b-8441-48c6-a6e7-07383e5746db" {{(pid=62499) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1049.450759] env[62499]: DEBUG nova.compute.manager [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62499) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1049.450838] env[62499]: DEBUG nova.compute.manager [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Deallocating network for instance {{(pid=62499) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1049.450975] env[62499]: DEBUG nova.network.neutron [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] deallocate_for_instance() {{(pid=62499) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1860}} [ 1049.465864] env[62499]: DEBUG nova.network.neutron [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Instance cache missing network info. {{(pid=62499) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3380}} [ 1049.968567] env[62499]: DEBUG nova.network.neutron [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Updating instance_info_cache with network_info: [] {{(pid=62499) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1050.471149] env[62499]: INFO nova.compute.manager [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] [instance: 6cdc499b-8441-48c6-a6e7-07383e5746db] Took 1.02 seconds to deallocate network for instance. [ 1051.499435] env[62499]: INFO nova.scheduler.client.report [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Deleted allocations for instance 6cdc499b-8441-48c6-a6e7-07383e5746db [ 1052.007061] env[62499]: DEBUG oslo_concurrency.lockutils [None req-619d1589-e774-47bb-ae60-3ac15c3f2d75 tempest-ServersTestJSON-497234563 tempest-ServersTestJSON-497234563-project-member] Lock "6cdc499b-8441-48c6-a6e7-07383e5746db" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 12.995s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1053.153471] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1053.153754] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1053.153959] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Starting heal instance info cache {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9928}} [ 1053.154103] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Rebuilding the list of instances to heal {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1053.659797] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Didn't find any instances for network info cache update. {{(pid=62499) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10014}} [ 1053.659797] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1053.660099] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1053.660099] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1053.660254] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1053.660413] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1053.660680] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1053.660793] env[62499]: DEBUG nova.compute.manager [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62499) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10547}} [ 1053.660948] env[62499]: DEBUG oslo_service.periodic_task [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Running periodic task ComputeManager.update_available_resource {{(pid=62499) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1054.164523] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1054.164903] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1054.164954] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1054.165092] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62499) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1054.165996] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c761a3e-d260-487e-9e0a-9678995895ce {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1054.174109] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4da9dfd8-46d2-42f6-ad53-cc10c9b65f8f {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1054.187484] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-22075bb3-15e7-46c5-88a7-eb838ffa5a38 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1054.193357] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab9e2653-e955-45e9-ba0d-178978a89fa2 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1054.221872] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181527MB free_disk=130GB free_vcpus=48 pci_devices=None {{(pid=62499) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1054.222055] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1054.222255] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1055.242388] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62499) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1055.242742] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62499) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1055.256197] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e4dbefa6-c7aa-46e9-a104-82b587b7b286 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1055.263491] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c036843-1f9d-439c-9294-77543c69c339 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1055.291721] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b32ca9e-ca2c-44fa-9628-aa0cbe237076 {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1055.298717] env[62499]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1720cedc-1ea4-441c-8e61-1db2a0b8c13b {{(pid=62499) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1055.311085] env[62499]: DEBUG nova.compute.provider_tree [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Inventory has not changed in ProviderTree for provider: 38a6c941-9755-44af-8da8-6ef6281d0157 {{(pid=62499) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1055.814412] env[62499]: DEBUG nova.scheduler.client.report [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Inventory has not changed for provider 38a6c941-9755-44af-8da8-6ef6281d0157 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 130, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62499) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1056.319788] env[62499]: DEBUG nova.compute.resource_tracker [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62499) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1056.320160] env[62499]: DEBUG oslo_concurrency.lockutils [None req-b3497e46-ee03-4825-80d4-8d085c2bccfd None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.098s {{(pid=62499) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}}